From 7043dd7317011fb0d8fae68f7be347258d6fa60a Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Sat, 14 May 2022 20:21:10 +0800 Subject: [PATCH 1/2] fix --- common/ast/src/parser/expr.rs | 2 +- common/ast/src/parser/token.rs | 4 +- common/ast/src/parser/util.rs | 41 ++++--- common/ast/tests/it/parser.rs | 1 + common/ast/tests/it/testdata/expr.txt | 101 +++++++++++++----- common/ast/tests/it/testdata/query-error.txt | 8 +- common/ast/tests/it/testdata/query.txt | 10 +- .../ast/tests/it/testdata/statement-error.txt | 2 +- common/ast/tests/it/testdata/statement.txt | 14 +-- common/ast/tests/it/token.rs | 8 +- .../05_ddl/05_0000_ddl_create_tables.sql | 2 +- .../09_0007_func_fuse_truncate_purge.sql | 2 +- .../09_0008_fuse_optimize_table.sql | 6 +- .../20+_others/20_0001_planner_v2.result | 9 +- .../20+_others/20_0001_planner_v2.sql | 1 + 15 files changed, 141 insertions(+), 70 deletions(-) diff --git a/common/ast/src/parser/expr.rs b/common/ast/src/parser/expr.rs index f34c69fd27fe..44d1b1a5dce6 100644 --- a/common/ast/src/parser/expr.rs +++ b/common/ast/src/parser/expr.rs @@ -750,9 +750,9 @@ pub fn expr_element(i: Input) -> IResult { | #case : "`CASE ... END`" | #exists : "`EXISTS (SELECT ...)`" | #subquery : "`(SELECT ...)`" - | #map_access : "[] | . | :" | #group | #column_ref : "" + | #map_access : "[] | . | :" ), )))(i)?; diff --git a/common/ast/src/parser/token.rs b/common/ast/src/parser/token.rs index bb7cbbf8edda..aaf06112b851 100644 --- a/common/ast/src/parser/token.rs +++ b/common/ast/src/parser/token.rs @@ -112,7 +112,7 @@ pub enum TokenKind { #[regex(r#"`[^`]*`"#)] #[regex(r#""([^"\\]|\\.|"")*""#)] #[regex(r#"'([^'\\]|\\.|'')*'"#)] - QuotedIdent, + QuotedString, #[regex(r"[xX]'[a-fA-F0-9]*'")] LiteralHex, @@ -608,7 +608,7 @@ impl TokenKind { !matches!( self, Ident - | QuotedIdent + | QuotedString | LiteralHex | LiteralNumber | DoubleEq diff --git a/common/ast/src/parser/util.rs b/common/ast/src/parser/util.rs index 189f50bc585d..85e4c01c861c 100644 --- a/common/ast/src/parser/util.rs +++ b/common/ast/src/parser/util.rs @@ -88,16 +88,27 @@ fn non_reserved_identifier( map( alt((rule! { Ident }, non_reserved_keyword(is_reserved_keyword))), |token| Identifier { + span: token.clone(), name: token.text().to_string(), quote: None, - span: token.clone(), }, ), - map(rule! { QuotedIdent }, |token| Identifier { - name: token.text()[1..token.text().len() - 1].to_string(), - quote: Some(token.text().chars().next().unwrap()), - span: token.clone(), - }), + move |i| { + match_token(QuotedString)(i).and_then(|(i2, token)| { + if token.text().starts_with('\'') { + Err(nom::Err::Error(Error::from_error_kind( + i, + ErrorKind::ExpectToken(Ident), + ))) + } else { + Ok((i2, Identifier { + span: token.clone(), + name: token.text()[1..token.text().len() - 1].to_string(), + quote: Some(token.text().chars().next().unwrap()), + })) + } + }) + }, ))(i) } } @@ -119,16 +130,20 @@ fn non_reserved_keyword( } pub fn literal_string(i: Input) -> IResult { - map( - rule! { - QuotedIdent - }, - |token| token.text()[1..token.text().len() - 1].to_string(), - )(i) + match_token(QuotedString)(i).and_then(|(i2, token)| { + if token.text().starts_with('\'') { + Ok((i2, token.text()[1..token.text().len() - 1].to_string())) + } else { + Err(nom::Err::Error(Error::from_error_kind( + i, + ErrorKind::ExpectToken(QuotedString), + ))) + } + }) } pub fn literal_u64(i: Input) -> IResult { - rule!(LiteralNumber)(i).and_then(|(i2, token)| { + match_token(LiteralNumber)(i).and_then(|(i2, token)| { token .text() .parse() diff --git a/common/ast/tests/it/parser.rs b/common/ast/tests/it/parser.rs index d65393c07cb1..152d59ac68df 100644 --- a/common/ast/tests/it/parser.rs +++ b/common/ast/tests/it/parser.rs @@ -247,6 +247,7 @@ fn test_expr() { r#"- - + + - 1 + + - 2"#, r#"1 + a * c.d"#, r#"number % 2"#, + r#"`t`:k1.k2"#, r#"col1 not between 1 and 2"#, r#"sum(col1)"#, r#""random"()"#, diff --git a/common/ast/tests/it/testdata/expr.txt b/common/ast/tests/it/testdata/expr.txt index 44297b9cc08e..c8052c2a22b4 100644 --- a/common/ast/tests/it/testdata/expr.txt +++ b/common/ast/tests/it/testdata/expr.txt @@ -343,6 +343,53 @@ BinaryOp { } +---------- Input ---------- +`t`:k1.k2 +---------- Output --------- +`t`:k1.k2 +---------- AST ------------ +MapAccess { + span: [ + Period(6..7), + Ident(7..9), + ], + expr: MapAccess { + span: [ + Colon(3..4), + Ident(4..6), + ], + expr: ColumnRef { + span: [ + QuotedString(0..3), + ], + database: None, + table: None, + column: Identifier { + name: "t", + quote: Some( + '`', + ), + span: QuotedString(0..3), + }, + }, + accessor: Colon { + key: Identifier { + name: "k1", + quote: None, + span: Ident(4..6), + }, + }, + }, + accessor: Period { + key: Identifier { + name: "k2", + quote: None, + span: Ident(7..9), + }, + }, +} + + ---------- Input ---------- col1 not between 1 and 2 ---------- Output --------- @@ -431,7 +478,7 @@ FunctionCall { ---------- AST ------------ FunctionCall { span: [ - QuotedIdent(0..8), + QuotedString(0..8), LParen(8..9), RParen(9..10), ], @@ -441,7 +488,7 @@ FunctionCall { quote: Some( '"', ), - span: QuotedIdent(0..8), + span: QuotedString(0..8), }, args: [], params: [], @@ -594,14 +641,14 @@ Trim { TRIM(0..4), LParen(4..5), LEADING(5..12), - QuotedIdent(13..18), + QuotedString(13..18), FROM(19..23), - QuotedIdent(24..29), + QuotedString(24..29), RParen(29..30), ], expr: Literal { span: [ - QuotedIdent(24..29), + QuotedString(24..29), ], lit: String( "def", @@ -612,7 +659,7 @@ Trim { Leading, Literal { span: [ - QuotedIdent(13..18), + QuotedString(13..18), ], lit: String( "abc", @@ -662,14 +709,14 @@ Position { span: [ POSITION(0..8), LParen(8..9), - QuotedIdent(9..12), + QuotedString(9..12), IN(13..15), Ident(16..19), RParen(19..20), ], substr_expr: Literal { span: [ - QuotedIdent(9..12), + QuotedString(9..12), ], lit: String( "a", @@ -900,7 +947,7 @@ arr[4]['k'] MapAccess { span: [ LBracket(6..7), - QuotedIdent(7..10), + QuotedString(7..10), RBracket(10..11), ], expr: MapAccess { @@ -959,7 +1006,7 @@ BinaryOp { }, right: Literal { span: [ - QuotedIdent(8..13), + QuotedString(8..13), ], lit: String( "^11", @@ -1141,7 +1188,7 @@ BinaryOp { Period(16..17), Ident(17..23), Eq(24..25), - QuotedIdent(26..35), + QuotedString(26..35), THEN(36..40), Ident(41..50), ELSE(51..55), @@ -1164,7 +1211,7 @@ BinaryOp { Period(16..17), Ident(17..23), Eq(24..25), - QuotedIdent(26..35), + QuotedString(26..35), THEN(36..40), Ident(41..50), ELSE(51..55), @@ -1200,7 +1247,7 @@ BinaryOp { }, right: Literal { span: [ - QuotedIdent(26..35), + QuotedString(26..35), ], lit: String( "GERMANY", @@ -1441,7 +1488,7 @@ BinaryOp { }, right: Literal { span: [ - QuotedIdent(48..58), + QuotedString(48..58), ], lit: String( "Brand#12", @@ -1453,13 +1500,13 @@ BinaryOp { span: [ IN(87..89), LParen(90..91), - QuotedIdent(91..100), + QuotedString(91..100), Comma(100..101), - QuotedIdent(102..110), + QuotedString(102..110), Comma(110..111), - QuotedIdent(112..121), + QuotedString(112..121), Comma(121..122), - QuotedIdent(123..131), + QuotedString(123..131), RParen(131..132), ], expr: ColumnRef { @@ -1477,7 +1524,7 @@ BinaryOp { list: [ Literal { span: [ - QuotedIdent(91..100), + QuotedString(91..100), ], lit: String( "SM CASE", @@ -1485,7 +1532,7 @@ BinaryOp { }, Literal { span: [ - QuotedIdent(102..110), + QuotedString(102..110), ], lit: String( "SM BOX", @@ -1493,7 +1540,7 @@ BinaryOp { }, Literal { span: [ - QuotedIdent(112..121), + QuotedString(112..121), ], lit: String( "SM PACK", @@ -1501,7 +1548,7 @@ BinaryOp { }, Literal { span: [ - QuotedIdent(123..131), + QuotedString(123..131), ], lit: String( "SM PKG", @@ -1681,9 +1728,9 @@ BinaryOp { span: [ IN(332..334), LParen(335..336), - QuotedIdent(336..341), + QuotedString(336..341), Comma(341..342), - QuotedIdent(343..352), + QuotedString(343..352), RParen(352..353), ], expr: ColumnRef { @@ -1701,7 +1748,7 @@ BinaryOp { list: [ Literal { span: [ - QuotedIdent(336..341), + QuotedString(336..341), ], lit: String( "AIR", @@ -1709,7 +1756,7 @@ BinaryOp { }, Literal { span: [ - QuotedIdent(343..352), + QuotedString(343..352), ], lit: String( "AIR REG", @@ -1738,7 +1785,7 @@ BinaryOp { }, right: Literal { span: [ - QuotedIdent(387..406), + QuotedString(387..406), ], lit: String( "DELIVER IN PERSON", diff --git a/common/ast/tests/it/testdata/query-error.txt b/common/ast/tests/it/testdata/query-error.txt index fde95ca06268..8557119e78d1 100644 --- a/common/ast/tests/it/testdata/query-error.txt +++ b/common/ast/tests/it/testdata/query-error.txt @@ -5,7 +5,7 @@ error: --> SQL:1:29 | 1 | select * from customer join where a = b - | ^^^^^ expected `(`, `SELECT`, , or + | ^^^^^ expected `(`, `SELECT`, , or ---------- Input ---------- @@ -15,7 +15,7 @@ error: --> SQL:1:15 | 1 | select * from join customer - | ------ ^^^^ expected `(`, `SELECT`, , or + | ------ ^^^^ expected `(`, `SELECT`, , or | | | while parsing `SELECT ...` @@ -27,7 +27,7 @@ error: --> SQL:1:30 | 1 | select * from t inner join t1 - | ^ expected `(`, `.`, , , `AS`, `ON`, or 1 more ... + | ^ expected `(`, `.`, , , `AS`, `ON`, or 1 more ... ---------- Input ---------- @@ -37,7 +37,7 @@ error: --> SQL:1:50 | 1 | select * from customer natural inner join orders on a = b - | ^^ expected `(`, `.`, , , `AS`, `INNER`, or 12 more ... + | ^^ expected `(`, `.`, , , `AS`, `INNER`, or 12 more ... ---------- Input ---------- diff --git a/common/ast/tests/it/testdata/query.txt b/common/ast/tests/it/testdata/query.txt index 007a36f9dcab..0f61cf9b7cc5 100644 --- a/common/ast/tests/it/testdata/query.txt +++ b/common/ast/tests/it/testdata/query.txt @@ -515,7 +515,7 @@ Query { Ident(394..403), NOT(404..407), LIKE(408..412), - QuotedIdent(413..422), + QuotedString(413..422), GROUP(443..448), BY(449..451), Ident(476..485), @@ -586,7 +586,7 @@ Query { Ident(394..403), NOT(404..407), LIKE(408..412), - QuotedIdent(413..422), + QuotedString(413..422), GROUP(443..448), BY(449..451), Ident(476..485), @@ -747,7 +747,7 @@ Query { Ident(394..403), NOT(404..407), LIKE(408..412), - QuotedIdent(413..422), + QuotedString(413..422), GROUP(443..448), BY(449..451), Ident(476..485), @@ -776,7 +776,7 @@ Query { Ident(394..403), NOT(404..407), LIKE(408..412), - QuotedIdent(413..422), + QuotedString(413..422), GROUP(443..448), BY(449..451), Ident(476..485), @@ -891,7 +891,7 @@ Query { }, right: Literal { span: [ - QuotedIdent(413..422), + QuotedString(413..422), ], lit: String( "%:1%:2%", diff --git a/common/ast/tests/it/testdata/statement-error.txt b/common/ast/tests/it/testdata/statement-error.txt index b640734fcaba..051354cbcd10 100644 --- a/common/ast/tests/it/testdata/statement-error.txt +++ b/common/ast/tests/it/testdata/statement-error.txt @@ -86,7 +86,7 @@ error: --> SQL:1:21 | 1 | insert into t format - | ------ ^ expected or + | ------ ^ expected or | | | while parsing `INSERT INTO [TABLE] [(, ...)] (FORMAT | VALUES | )` diff --git a/common/ast/tests/it/testdata/statement.txt b/common/ast/tests/it/testdata/statement.txt index 525f2829d732..ddea4175d47c 100644 --- a/common/ast/tests/it/testdata/statement.txt +++ b/common/ast/tests/it/testdata/statement.txt @@ -418,7 +418,7 @@ TruncateTable { quote: Some( '"', ), - span: QuotedIdent(15..18), + span: QuotedString(15..18), }, ), table: Identifier { @@ -465,7 +465,7 @@ DropTable { quote: Some( '"', ), - span: QuotedIdent(23..26), + span: QuotedString(23..26), }, } @@ -481,7 +481,7 @@ UseDatabase { quote: Some( '"', ), - span: QuotedIdent(4..7), + span: QuotedString(4..7), }, } @@ -2444,7 +2444,7 @@ Query( SELECT(0..6), Ident(7..17), LParen(17..18), - QuotedIdent(18..37), + QuotedString(18..37), RParen(37..38), Period(38..39), Ident(39..41), @@ -2458,7 +2458,7 @@ Query( SELECT(0..6), Ident(7..17), LParen(17..18), - QuotedIdent(18..37), + QuotedString(18..37), RParen(37..38), Period(38..39), Ident(39..41), @@ -2484,7 +2484,7 @@ Query( span: [ Ident(7..17), LParen(17..18), - QuotedIdent(18..37), + QuotedString(18..37), RParen(37..38), ], distinct: false, @@ -2496,7 +2496,7 @@ Query( args: [ Literal { span: [ - QuotedIdent(18..37), + QuotedString(18..37), ], lit: String( "{\"k1\": [0, 1, 2]}", diff --git a/common/ast/tests/it/token.rs b/common/ast/tests/it/token.rs index 05219e458d5e..e40b80c44d96 100644 --- a/common/ast/tests/it/token.rs +++ b/common/ast/tests/it/token.rs @@ -25,7 +25,7 @@ fn test_lexer() { &[ (LiteralHex, "x'deadbeef'", 0..11), ( - QuotedIdent, + QuotedString, "'a string literal\n escape quote by '' or \\'. '", 29..75, ), @@ -33,8 +33,8 @@ fn test_lexer() { ], ); assert_lex("'中文' '日本語'", &[ - (QuotedIdent, "'中文'", 0..8), - (QuotedIdent, "'日本語'", 9..20), + (QuotedString, "'中文'", 0..8), + (QuotedString, "'日本語'", 9..20), (EOI, "", 20..20), ]); assert_lex("42 3.5 4. .001 5e2 1.925e-3 .38e+7 1.e-01", &[ @@ -53,7 +53,7 @@ fn test_lexer() { &[ (CREATE, "create", 0..6), (TABLE, "table", 7..12), - (QuotedIdent, "\"user\"", 13..19), + (QuotedString, "\"user\"", 13..19), (LParen, "(", 20..21), (Ident, "id", 21..23), (INT, "int", 24..27), diff --git a/tests/suites/0_stateless/05_ddl/05_0000_ddl_create_tables.sql b/tests/suites/0_stateless/05_ddl/05_0000_ddl_create_tables.sql index 4e43dcd9f60c..764a1ed158cd 100644 --- a/tests/suites/0_stateless/05_ddl/05_0000_ddl_create_tables.sql +++ b/tests/suites/0_stateless/05_ddl/05_0000_ddl_create_tables.sql @@ -20,7 +20,7 @@ create table t3(a int,b int) engine=Memory CLUSTER BY(a); -- {ErrorCode 2703} create table t3(`a` int) ENGINE = Null; -create table t4('a' int) ENGINE = Null; +create table t4(a int) ENGINE = Null; DROP TABLE IF EXISTS t; DROP TABLE IF EXISTS t2; diff --git a/tests/suites/0_stateless/09_fuse_engine/09_0007_func_fuse_truncate_purge.sql b/tests/suites/0_stateless/09_fuse_engine/09_0007_func_fuse_truncate_purge.sql index 9976f219e8b2..9e0d5070df3e 100644 --- a/tests/suites/0_stateless/09_fuse_engine/09_0007_func_fuse_truncate_purge.sql +++ b/tests/suites/0_stateless/09_fuse_engine/09_0007_func_fuse_truncate_purge.sql @@ -13,7 +13,7 @@ select count(*) from fuse_snapshot('db_09_0007', 't'); -- truncate table will remove all the historical data but the latest snapshot -- only the latest snapshot will be kept, no segments or block, but unfortunately, we can not verify it by sql (yet) -truncate table 't' purge; +truncate table `t` purge; -- expect 1 snapshot left select count(*) from fuse_snapshot('db_09_0007', 't'); -- but no data, since it is truncated diff --git a/tests/suites/0_stateless/09_fuse_engine/09_0008_fuse_optimize_table.sql b/tests/suites/0_stateless/09_fuse_engine/09_0008_fuse_optimize_table.sql index e652b079a011..e644e137b2ac 100644 --- a/tests/suites/0_stateless/09_fuse_engine/09_0008_fuse_optimize_table.sql +++ b/tests/suites/0_stateless/09_fuse_engine/09_0008_fuse_optimize_table.sql @@ -22,13 +22,13 @@ select count(*)=4 from fuse_snapshot('db_09_0008', 't'); --------------------------- -- purge data and history -optimize table 't' purge; +optimize table `t` purge; -- expect 1 snapshot left select count(*)=1 from fuse_snapshot('db_09_0008', 't'); -- check the data select * from t order by a; -- purge again, nothing happens -optimize table 't' purge; +optimize table `t` purge; select * from t order by a; --------------------------- @@ -38,7 +38,7 @@ insert into t values (9); insert into t values (10); select * from t order by a; -- purge and compact -optimize table 't' all; +optimize table `t` all; -- expect 1 snapshot left select count(*)=1 from fuse_snapshot('db_09_0008', 't'); select * from t order by a; diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result index 3d8008f50694..87ea071f0bd8 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result @@ -29,6 +29,7 @@ ====MAP_ACCESS==== 2 2 +2 ====AGGREGATOR==== 2 4 @@ -153,7 +154,13 @@ NULL 2 3 ====SELECT_WITHOUT_FROM==== 2 8 -new_planner +ERROR 1105 (HY000) at line 119: Code: 1065, displayText = error: + --> SQL:1:8 + | +1 | select "new_planner" + | ^^^^^^^^^^^^^ column doesn't exist + +. === Test limit === 0 1 diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql index c0be334510d3..012fdbfdc9d7 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql @@ -35,6 +35,7 @@ select * from numbers(5) where number in (1, 3); select '====MAP_ACCESS===='; select parse_json('{"k1": [0, 1, 2]}'):k1[2]; select parse_json('{"k1": [0, 1, 2]}')['k1'][2]; +select parse_json('{"k1": {"k2": [0, 1, 2]}}'):k1.k2[2]; -- Aggregator operator select '====AGGREGATOR===='; From f0aeb766ed7d596f91685d9d2e007121c024aef2 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Sat, 14 May 2022 23:24:56 +0800 Subject: [PATCH 2/2] fix --- .../0_stateless/20+_others/20_0001_planner_v2.result | 8 +------- .../suites/0_stateless/20+_others/20_0001_planner_v2.sql | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result index 87ea071f0bd8..c928a8ba5810 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result @@ -154,13 +154,7 @@ NULL 2 3 ====SELECT_WITHOUT_FROM==== 2 8 -ERROR 1105 (HY000) at line 119: Code: 1065, displayText = error: - --> SQL:1:8 - | -1 | select "new_planner" - | ^^^^^^^^^^^^^ column doesn't exist - -. +new_planner === Test limit === 0 1 diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql index 012fdbfdc9d7..8519e7778c8d 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql @@ -134,7 +134,7 @@ drop table t3; select '====SELECT_WITHOUT_FROM===='; select 1 + 1; select to_int(8); -select "new_planner"; +select 'new_planner'; select *; -- {ErrorCode 1065} -- limit