From d7d7720bb9a55b30e3a803610c50616be559ba39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E6=9E=B8?= Date: Mon, 9 Dec 2024 15:30:07 +0800 Subject: [PATCH 1/2] Fix parse array with data type. --- .../druid/sql/ast/expr/SQLArrayExpr.java | 27 ++++++---- .../druid/sql/parser/SQLExprParser.java | 37 +++++++++++--- .../sql/visitor/SQLASTOutputVisitor.java | 12 +++-- .../bvt/sql/bigquery/BigQueryDirTest.java | 50 ++++++++++++------- .../test/resources/bvt/parser/bigquery/0.txt | 29 +++++++++-- 5 files changed, 109 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLArrayExpr.java b/core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLArrayExpr.java index 48ed0e097a..f6903757cd 100644 --- a/core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLArrayExpr.java +++ b/core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLArrayExpr.java @@ -15,6 +15,7 @@ */ package com.alibaba.druid.sql.ast.expr; +import com.alibaba.druid.sql.ast.SQLDataType; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLExprImpl; import com.alibaba.druid.sql.ast.SQLObject; @@ -26,16 +27,28 @@ public class SQLArrayExpr extends SQLExprImpl implements SQLReplaceable { private SQLExpr expr; + private SQLDataType dataType; private List values = new ArrayList(); public SQLArrayExpr() { } + public SQLDataType getDataType() { + return dataType; + } + + public void setDataType(SQLDataType dataType) { + this.dataType = dataType; + } + public SQLArrayExpr clone() { SQLArrayExpr x = new SQLArrayExpr(); if (expr != null) { x.setExpr(expr.clone()); } + if (dataType != null) { + x.setDataType(dataType.clone()); + } for (SQLExpr value : values) { SQLExpr value2 = value.clone(); value2.setParent(x); @@ -76,17 +89,9 @@ public void setValues(List values) { @Override protected void accept0(SQLASTVisitor visitor) { if (visitor.visit(this)) { - if (expr != null) { - this.expr.accept(visitor); - } - - if (values != null) { - for (SQLExpr value : values) { - if (value != null) { - value.accept(visitor); - } - } - } + acceptChild(visitor, expr); + acceptChild(visitor, dataType); + acceptChild(visitor, values); } visitor.endVisit(this); } diff --git a/core/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/core/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index 47886440d4..3a08c316bf 100644 --- a/core/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/core/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -586,12 +586,21 @@ && dialectFeatureEnabled(SQLTimestampExpr)) { } else if (hash_lower == FnvHash.Constants.TIMESTAMP && lexer.token == Token.LITERAL_ALIAS) { sqlExpr = new SQLTimestampExpr(lexer.stringVal()); lexer.nextToken(); - } else if (hash_lower == FnvHash.Constants.ARRAY && lexer.token == Token.LBRACKET) { + } else if (hash_lower == FnvHash.Constants.ARRAY && (lexer.token == Token.LBRACKET || lexer.token == Token.LT)) { SQLArrayExpr array = new SQLArrayExpr(); array.setExpr(new SQLIdentifierExpr(ident)); - accept(Token.LBRACKET); - this.exprList(array.getValues(), array); - accept(Token.RBRACKET); + if (lexer.nextIf(Token.LT)) { + SQLDataType sqlDataType = this.parseDataType(); + array.setDataType(sqlDataType); + accept(Token.GT); + } + + if (lexer.nextIf(Token.LBRACKET)) { + this.exprList(array.getValues(), array); + accept(Token.RBRACKET); + } else { + throw new ParserException("Syntax error. " + lexer.info()); + } sqlExpr = array; } else { sqlExpr = primaryIdentifierRest(hash_lower, ident); @@ -1241,10 +1250,18 @@ && dialectFeatureEnabled(SQLTimestampExpr)) { SQLArrayExpr array = new SQLArrayExpr(); array.setExpr(new SQLIdentifierExpr("ARRAY")); lexer.nextToken(); + if (lexer.nextIf(Token.LT)) { + SQLDataType sqlDataType = this.parseDataType(); + array.setDataType(sqlDataType); + accept(Token.GT); + } + if (lexer.nextIf(Token.LBRACKET)) { + this.exprList(array.getValues(), array); + accept(Token.RBRACKET); + } else { + throw new ParserException("Syntax error. " + lexer.info()); + } - accept(Token.LBRACKET); - this.exprList(array.getValues(), array); - accept(Token.RBRACKET); sqlExpr = array; break; } @@ -6404,7 +6421,11 @@ protected SQLStructExpr struct() { SQLStructExpr structExpr = new SQLStructExpr(); accept(Token.LPAREN); aliasedItems(structExpr.getItems(), structExpr); - accept(Token.RPAREN); + if (lexer.token == Token.GTGT) { + lexer.token = Token.RPAREN; + } else { + accept(Token.RPAREN); + } return structExpr; } } diff --git a/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index f12c5d6389..40dc48bbc4 100644 --- a/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -4893,11 +4893,7 @@ public boolean visit(SQLUnnestTableSource x) { final List columns = x.getColumns(); final String alias = x.getAlias(); if (alias != null) { - if (columns.size() > 0) { - print0(ucase ? " AS " : " as "); - } else { - print(' '); - } + print0(ucase ? " AS " : " as "); print0(alias); } @@ -7511,6 +7507,12 @@ public boolean visit(SQLArrayExpr x) { } } + if (x.getDataType() != null) { + print('<'); + x.getDataType().accept(this); + print('>'); + } + boolean brace = arrayValue && (dbType == DbType.hive || dbType == DbType.spark || dbType == DbType.odps); print(brace ? '(' : '['); diff --git a/core/src/test/java/com/alibaba/druid/bvt/sql/bigquery/BigQueryDirTest.java b/core/src/test/java/com/alibaba/druid/bvt/sql/bigquery/BigQueryDirTest.java index 646c34f89d..43f32b4834 100644 --- a/core/src/test/java/com/alibaba/druid/bvt/sql/bigquery/BigQueryDirTest.java +++ b/core/src/test/java/com/alibaba/druid/bvt/sql/bigquery/BigQueryDirTest.java @@ -4,6 +4,7 @@ import com.alibaba.druid.bvt.sql.SQLResourceTest; import com.alibaba.druid.sql.SQLUtils; import org.apache.commons.io.FileUtils; +import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -20,25 +21,38 @@ protected File dir(String path) { return new File(path); } + @Ignore @Test public void dirTest() throws Exception { -// File dir = new File("/Users/wenshao/Downloads/goto_1894_sql"); -// File dir = new File("/Users/wenshao/Downloads/BigQuery"); -// File[] files = dir.listFiles(); -// Arrays.sort(files, Comparator.comparing(e -> e.getName().toLowerCase())); -// for (File file : files) { -// if (file.getName().equals(".DS_Store")) { -// continue; -// } -// -// System.out.println(file.getAbsolutePath()); -// String sql = FileUtils.readFileToString(file, StandardCharsets.UTF_8); -// try { -// SQLUtils.parseStatements(sql, DbType.bigquery); -// file.delete(); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// } + // File dir = new File("/Users/wenshao/Downloads/goto_1894_sql"); + File dir = new File("/Users/lingo/workspace/alibaba/druid/goto/all_etl_jobs/"); + File[] files = dir.listFiles(); + Arrays.sort(files, Comparator.comparing(e -> e.getName().toLowerCase())); + long total = files.length; + long success = 0L; + for (File file : files) { + if (file.getName().equals(".DS_Store")) { + continue; + } + + + String sql = FileUtils.readFileToString(file, StandardCharsets.UTF_8); + try { + SQLUtils.parseStatements(sql, DbType.bigquery); + success += 1; + } catch (Exception e) { + e.printStackTrace(); + System.out.println(file.getAbsolutePath()); + } + } + System.out.println("success: " + success + "/" + total); + } + + @Ignore + @Test + public void fileTest() throws Exception { + File file = new File("/Users/lingo/workspace/alibaba/druid/goto/all_etl_jobs/gojek-mart-kafka-gofood_challenges_campaign.sql"); + String sql = FileUtils.readFileToString(file, StandardCharsets.UTF_8); + SQLUtils.parseStatements(sql, DbType.bigquery); } } diff --git a/core/src/test/resources/bvt/parser/bigquery/0.txt b/core/src/test/resources/bvt/parser/bigquery/0.txt index c4e308f44b..cff715ec02 100644 --- a/core/src/test/resources/bvt/parser/bigquery/0.txt +++ b/core/src/test/resources/bvt/parser/bigquery/0.txt @@ -1,3 +1,24 @@ + SELECT + campaign.key AS campaign_key, + campaign.start_date AS campaign_start_date, + campaign.end_date AS campaign_end_date, + FROM + UNNEST([ + STRUCT( + 'trial_campaign' AS key, + '2024-11-17T17:00:00+00:00' AS start_date, + '2024-12-01T16:59:59+00:00' AS end_date, + ARRAY>[ + STRUCT('mission_1', '2024-11-17T17:00:00+00:00', '2024-11-24T16:59:59+00:00'), + STRUCT('mission_2', '2024-11-24T17:00:00+00:00', '2024-12-01T16:59:59+00:00') + ] AS missions + ) + ]) AS campaign +-------------------- +SELECT campaign.key AS campaign_key, campaign.start_date AS campaign_start_date, campaign.end_date AS campaign_end_date +FROM UNNEST([STRUCT('trial_campaign' AS key, '2024-11-17T17:00:00+00:00' AS start_date, '2024-12-01T16:59:59+00:00' AS end_date, ARRAY>[STRUCT('mission_1', '2024-11-17T17:00:00+00:00', '2024-11-24T16:59:59+00:00'), + STRUCT('mission_2', '2024-11-24T17:00:00+00:00', '2024-12-01T16:59:59+00:00')] AS missions)]) AS campaign +------------------------------------------------------------------------------------------------------------------------ CREATE TEMP FUNCTION invalidRoute(polyline1 STRING) RETURNS BOOL LANGUAGE js AS @@ -216,7 +237,7 @@ WITH Races AS ( STRUCT('Lewandowski' AS name, [25.0, 25.7, 26.3, 27.2] AS laps), STRUCT('Kipketer' AS name, [23.2, 26.1, 27.3, 29.4] AS laps), STRUCT('Berian' AS name, [23.7, 26.1, 27.0, 29.3] AS laps), - STRUCT('Nathan' AS name, ARRAY < FLOAT64 > [] AS laps), + STRUCT('Nathan' AS name, ARRAY[] AS laps), STRUCT('David' AS name, NULL AS laps)] AS participants ) SELECT name, sum(duration) AS finish_time @@ -564,7 +585,7 @@ FROM UNNEST([2, 1, -2, 3, -2, 1, 2]) AS x; -------------------- SELECT ARRAY_AGG(x LIMIT 5) AS array_agg FROM UNNEST([2, 1, -2, 3, -2, - 1, 2]) x; + 1, 2]) AS x; ------------------------------------------------------------------------------------------------------------------------ ASSERT ( (SELECT COUNT(*) > 5 FROM UNNEST([1, 2, 3, 4, 5, 6])) @@ -687,6 +708,6 @@ SELECT FROM professors GROUP BY item; ------------------------------------------------------------------------------------------------------------------------ -select cast(json_parse(tickets_fields_value) as array(varchar)) as tickets_fields_value +select cast(json_parse(tickets_fields_value) as array) as tickets_fields_value -------------------- -SELECT CAST(json_parse(tickets_fields_value) AS ARRAY(varchar)) AS tickets_fields_value \ No newline at end of file +SELECT CAST(json_parse(tickets_fields_value) AS ARRAY) AS tickets_fields_value \ No newline at end of file From 6fecd032c0c2d98167af07b45e65006de640bee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E6=9E=B8?= Date: Mon, 9 Dec 2024 16:09:39 +0800 Subject: [PATCH 2/2] Revert test. --- .../bvt/sql/bigquery/BigQueryDirTest.java | 53 +++++++------------ 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/core/src/test/java/com/alibaba/druid/bvt/sql/bigquery/BigQueryDirTest.java b/core/src/test/java/com/alibaba/druid/bvt/sql/bigquery/BigQueryDirTest.java index 43f32b4834..b994d93d85 100644 --- a/core/src/test/java/com/alibaba/druid/bvt/sql/bigquery/BigQueryDirTest.java +++ b/core/src/test/java/com/alibaba/druid/bvt/sql/bigquery/BigQueryDirTest.java @@ -2,15 +2,10 @@ import com.alibaba.druid.DbType; import com.alibaba.druid.bvt.sql.SQLResourceTest; -import com.alibaba.druid.sql.SQLUtils; -import org.apache.commons.io.FileUtils; import org.junit.Ignore; import org.junit.Test; import java.io.File; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Comparator; public class BigQueryDirTest extends SQLResourceTest { public BigQueryDirTest() { @@ -24,35 +19,23 @@ protected File dir(String path) { @Ignore @Test public void dirTest() throws Exception { - // File dir = new File("/Users/wenshao/Downloads/goto_1894_sql"); - File dir = new File("/Users/lingo/workspace/alibaba/druid/goto/all_etl_jobs/"); - File[] files = dir.listFiles(); - Arrays.sort(files, Comparator.comparing(e -> e.getName().toLowerCase())); - long total = files.length; - long success = 0L; - for (File file : files) { - if (file.getName().equals(".DS_Store")) { - continue; - } - - - String sql = FileUtils.readFileToString(file, StandardCharsets.UTF_8); - try { - SQLUtils.parseStatements(sql, DbType.bigquery); - success += 1; - } catch (Exception e) { - e.printStackTrace(); - System.out.println(file.getAbsolutePath()); - } - } - System.out.println("success: " + success + "/" + total); - } - - @Ignore - @Test - public void fileTest() throws Exception { - File file = new File("/Users/lingo/workspace/alibaba/druid/goto/all_etl_jobs/gojek-mart-kafka-gofood_challenges_campaign.sql"); - String sql = FileUtils.readFileToString(file, StandardCharsets.UTF_8); - SQLUtils.parseStatements(sql, DbType.bigquery); +// File dir = new File("/Users/wenshao/Downloads/goto_1894_sql"); +// File dir = new File("/Users/wenshao/Downloads/BigQuery"); +// File[] files = dir.listFiles(); +// Arrays.sort(files, Comparator.comparing(e -> e.getName().toLowerCase())); +// for (File file : files) { +// if (file.getName().equals(".DS_Store")) { +// continue; +// } +// +// System.out.println(file.getAbsolutePath()); +// String sql = FileUtils.readFileToString(file, StandardCharsets.UTF_8); +// try { +// SQLUtils.parseStatements(sql, DbType.bigquery); +// file.delete(); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } } }