diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 84a858e8b2242..14eded05156cf 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1128,6 +1128,18 @@ ], "sqlState" : "42614" }, + "DUPLICATE_CONDITION_NAME_FOR_DIFFERENT_SQL_STATE" : { + "message" : [ + "Found duplicate condition name for different SQL states. Please, remove one of them." + ], + "sqlState" : "42710" + }, + "DUPLICATE_HANDLER_FOR_SAME_SQL_STATE" : { + "message" : [ + "Found duplicate handlers for the same SQL state . Please, remove one of them." + ], + "sqlState" : "42710" + }, "DUPLICATE_KEY" : { "message" : [ "Found duplicate keys ." @@ -1152,6 +1164,12 @@ }, "sqlState" : "4274K" }, + "DUPLICATE_SQL_STATE_FOR_SAME_HANDLER" : { + "message" : [ + "Found duplicate SQL state for the same handler. Please, remove one of them." + ], + "sqlState" : "42710" + }, "EMITTING_ROWS_OLDER_THAN_WATERMARK_NOT_ALLOWED" : { "message" : [ "Previous node emitted a row with eventTime= which is older than current_watermark_value=", @@ -4830,6 +4848,11 @@ " is a VARIABLE and cannot be updated using the SET statement. Use SET VARIABLE = ... instead." ] }, + "SQL_SCRIPTING_NOT_ENABLED" : { + "message" : [ + "SQL scripting is under development and not all features are supported. To enable existing features set to `true`." + ] + }, "STATE_STORE_MULTIPLE_COLUMN_FAMILIES" : { "message" : [ "Creating multiple column families with is not supported." diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md index f5e1ddfd3c576..ff9de533cf0aa 100644 --- a/docs/sql-ref-ansi-compliance.md +++ b/docs/sql-ref-ansi-compliance.md @@ -452,8 +452,10 @@ Below is a list of all the keywords in Spark SQL. |COMPENSATION|non-reserved|non-reserved|non-reserved| |COMPUTE|non-reserved|non-reserved|non-reserved| |CONCATENATE|non-reserved|non-reserved|non-reserved| +|CONDITION|reserved|non-reserved|reserved| |CONSTRAINT|reserved|non-reserved|reserved| |CONTAINS|non-reserved|non-reserved|non-reserved| +|CONTINUE|non-reserved|non-reserved|non-reserved| |COST|non-reserved|non-reserved|non-reserved| |CREATE|reserved|non-reserved|reserved| |CROSS|reserved|strict-non-reserved|reserved| @@ -505,6 +507,7 @@ Below is a list of all the keywords in Spark SQL. |EXCLUDE|non-reserved|non-reserved|non-reserved| |EXECUTE|reserved|non-reserved|reserved| |EXISTS|non-reserved|non-reserved|reserved| +|EXIT|non-reserved|non-reserved|non-reserved| |EXPLAIN|non-reserved|non-reserved|non-reserved| |EXPORT|non-reserved|non-reserved|non-reserved| |EXTENDED|non-reserved|non-reserved|non-reserved| @@ -522,6 +525,7 @@ Below is a list of all the keywords in Spark SQL. |FOREIGN|reserved|non-reserved|reserved| |FORMAT|non-reserved|non-reserved|non-reserved| |FORMATTED|non-reserved|non-reserved|non-reserved| +|FOUND|non-reserved|non-reserved|non-reserved| |FROM|reserved|non-reserved|reserved| |FULL|reserved|strict-non-reserved|reserved| |FUNCTION|non-reserved|non-reserved|reserved| @@ -531,6 +535,7 @@ Below is a list of all the keywords in Spark SQL. |GRANT|reserved|non-reserved|reserved| |GROUP|reserved|non-reserved|reserved| |GROUPING|non-reserved|non-reserved|reserved| +|HANDLER|non-reserved|non-reserved|non-reserved| |HAVING|reserved|non-reserved|reserved| |HOUR|non-reserved|non-reserved|non-reserved| |HOURS|non-reserved|non-reserved|non-reserved| @@ -684,6 +689,7 @@ Below is a list of all the keywords in Spark SQL. |SOURCE|non-reserved|non-reserved|non-reserved| |SPECIFIC|non-reserved|non-reserved|reserved| |SQL|reserved|non-reserved|reserved| +|SQLEXCEPTION|non-reserved|non-reserved|non-reserved| |START|non-reserved|non-reserved|reserved| |STATISTICS|non-reserved|non-reserved|non-reserved| |STORED|non-reserved|non-reserved|non-reserved| diff --git a/python/pyspark/errors/error-conditions.json b/python/pyspark/errors/error-conditions.json index 4061d024a83cd..6e4c53d91a8e8 100644 --- a/python/pyspark/errors/error-conditions.json +++ b/python/pyspark/errors/error-conditions.json @@ -913,6 +913,11 @@ "Slice with step is not supported." ] }, + "SQL_SCRIPTING_NOT_ENABLED": { + "message": [ + "SQL Scripting is under development and not all features are supported. To enable existing features set to `true`." + ] + }, "STATE_NOT_EXISTS": { "message": [ "State is either not defined or has already been removed." diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 index acfc0011f5d05..a717911f3843c 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 @@ -172,8 +172,10 @@ COMPACTIONS: 'COMPACTIONS'; COMPENSATION: 'COMPENSATION'; COMPUTE: 'COMPUTE'; CONCATENATE: 'CONCATENATE'; +CONDITION: 'CONDITION'; CONSTRAINT: 'CONSTRAINT'; CONTAINS: 'CONTAINS'; +CONTINUE: 'CONTINUE'; COST: 'COST'; CREATE: 'CREATE'; CROSS: 'CROSS'; @@ -224,6 +226,7 @@ EXCEPT: 'EXCEPT'; EXCHANGE: 'EXCHANGE'; EXCLUDE: 'EXCLUDE'; EXISTS: 'EXISTS'; +EXIT: 'EXIT'; EXPLAIN: 'EXPLAIN'; EXPORT: 'EXPORT'; EXTENDED: 'EXTENDED'; @@ -241,6 +244,7 @@ FOR: 'FOR'; FOREIGN: 'FOREIGN'; FORMAT: 'FORMAT'; FORMATTED: 'FORMATTED'; +FOUND: 'FOUND'; FROM: 'FROM'; FULL: 'FULL'; FUNCTION: 'FUNCTION'; @@ -250,6 +254,7 @@ GLOBAL: 'GLOBAL'; GRANT: 'GRANT'; GROUP: 'GROUP'; GROUPING: 'GROUPING'; +HANDLER: 'HANDLER'; HAVING: 'HAVING'; BINARY_HEX: 'X'; HOUR: 'HOUR'; @@ -403,6 +408,7 @@ SORTED: 'SORTED'; SOURCE: 'SOURCE'; SPECIFIC: 'SPECIFIC'; SQL: 'SQL'; +SQLEXCEPTION: 'SQLEXCEPTION'; START: 'START'; STATISTICS: 'STATISTICS'; STORED: 'STORED'; diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index 5b8805821b045..f851782021627 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -63,6 +63,8 @@ compoundStatement : statement | setStatementWithOptionalVarKeyword | beginEndCompoundBlock + | declareCondition + | declareHandler | ifElseStatement | whileStatement ; @@ -87,6 +89,23 @@ singleStatement : (statement|setResetStatement) SEMICOLON* EOF ; +conditionValue + : stringLit + | multipartIdentifier + ; + +conditionValueList + : ((conditionValues+=conditionValue (COMMA conditionValues+=conditionValue)*) | SQLEXCEPTION | NOT FOUND) + ; + +declareCondition + : DECLARE multipartIdentifier CONDITION (FOR stringLit)? + ; + +declareHandler + : DECLARE (CONTINUE | EXIT) HANDLER FOR conditionValueList (BEGIN compoundBody END | statement | setStatementWithOptionalVarKeyword) + ; + beginLabel : multipartIdentifier COLON ; @@ -1503,6 +1522,7 @@ ansiNonReserved | COMPUTE | CONCATENATE | CONTAINS + | CONTINUE | COST | CUBE | CURRENT @@ -1542,6 +1562,7 @@ ansiNonReserved | EXCHANGE | EXCLUDE | EXISTS + | EXIT | EXPLAIN | EXPORT | EXTENDED @@ -1554,11 +1575,13 @@ ansiNonReserved | FOLLOWING | FORMAT | FORMATTED + | FOUND | FUNCTION | FUNCTIONS | GENERATED | GLOBAL | GROUPING + | HANDLER | HOUR | HOURS | IDENTIFIER_KW @@ -1684,6 +1707,7 @@ ansiNonReserved | SORTED | SOURCE | SPECIFIC + | SQLEXCEPTION | START | STATISTICS | STORED @@ -1828,8 +1852,10 @@ nonReserved | COMPENSATION | COMPUTE | CONCATENATE + | CONDITION | CONSTRAINT | CONTAINS + | CONTINUE | COST | CREATE | CUBE @@ -1879,6 +1905,7 @@ nonReserved | EXCLUDE | EXECUTE | EXISTS + | EXIT | EXPLAIN | EXPORT | EXTENDED @@ -1896,6 +1923,7 @@ nonReserved | FOREIGN | FORMAT | FORMATTED + | FOUND | FROM | FUNCTION | FUNCTIONS @@ -1904,6 +1932,7 @@ nonReserved | GRANT | GROUP | GROUPING + | HANDLER | HAVING | HOUR | HOURS @@ -2047,6 +2076,7 @@ nonReserved | SOURCE | SPECIFIC | SQL + | SQLEXCEPTION | START | STATISTICS | STORED diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index c6e0467b3aff2..70ca00665c846 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.parser import java.util.Locale import java.util.concurrent.TimeUnit +import scala.collection.mutable import scala.collection.mutable.{ArrayBuffer, ListBuffer, Set} import scala.jdk.CollectionConverters._ import scala.util.{Left, Right} @@ -119,11 +120,13 @@ class AstBuilder extends DataTypeAstBuilder override def visitCompoundOrSingleStatement( ctx: CompoundOrSingleStatementContext): CompoundBody = withOrigin(ctx) { Option(ctx.singleCompoundStatement()).map { s => + if (!SQLConf.get.sqlScriptingEnabled) { + throw SqlScriptingErrors.sqlScriptingNotEnabled(CurrentOrigin.get) + } visit(s).asInstanceOf[CompoundBody] }.getOrElse { val logicalPlan = visitSingleStatement(ctx.singleStatement()) - CompoundBody(Seq(SingleStatement(parsedPlan = logicalPlan)), - Some(java.util.UUID.randomUUID.toString.toLowerCase(Locale.ROOT))) + CompoundBody(Seq(SingleStatement(parsedPlan = logicalPlan))) } } @@ -136,8 +139,24 @@ class AstBuilder extends DataTypeAstBuilder label: Option[String], allowVarDeclare: Boolean): CompoundBody = { val buff = ListBuffer[CompoundPlanStatement]() + val handlers = ListBuffer[ErrorHandler]() + val conditions = mutable.HashMap[String, String]() + val sqlStates = mutable.Set[String]() + ctx.compoundStatements.forEach(compoundStatement => { - buff += visit(compoundStatement).asInstanceOf[CompoundPlanStatement] + val stmt = visit(compoundStatement).asInstanceOf[CompoundPlanStatement] + + stmt match { + case handler: ErrorHandler => handlers += handler + case condition: ErrorCondition => + if (conditions.contains(condition.conditionName)) { + throw SqlScriptingErrors.duplicateConditionNameForDifferentSqlState( + CurrentOrigin.get, condition.conditionName) + } + conditions += condition.conditionName -> condition.value + sqlStates += condition.value + case s => buff += s + } }) val compoundStatements = buff.toList @@ -171,7 +190,7 @@ class AstBuilder extends DataTypeAstBuilder case _ => } - CompoundBody(buff.toSeq, label) + CompoundBody(buff.toSeq, label, handlers.toSeq, conditions) } @@ -220,6 +239,27 @@ class AstBuilder extends DataTypeAstBuilder } } + override def visitConditionValue(ctx: ConditionValueContext): String = { + Option(ctx.multipartIdentifier()).map(_.getText) + .getOrElse(ctx.stringLit().getText).replace("'", "") + } + + override def visitConditionValueList(ctx: ConditionValueListContext): Seq[String] = { + Option(ctx.SQLEXCEPTION()).map(_ => Seq("SQLEXCEPTION")).getOrElse { + Option(ctx.NOT()).map(_ => Seq("NOT FOUND")).getOrElse { + val buff = scala.collection.mutable.Set[String]() + ctx.conditionValues.forEach { conditionValue => + val elem = visit(conditionValue).asInstanceOf[String] + if (buff(elem)) { + throw SqlScriptingErrors.duplicateSqlStateForSameHandler(CurrentOrigin.get, elem) + } + buff += elem + } + buff.toSeq + } + } + } + override def visitIfElseStatement(ctx: IfElseStatementContext): IfElseStatement = { IfElseStatement( conditions = ctx.booleanExpression().asScala.toList.map(boolExpr => withOrigin(boolExpr) { @@ -247,6 +287,28 @@ class AstBuilder extends DataTypeAstBuilder WhileStatement(condition, body, Some(labelText)) } + override def visitDeclareCondition(ctx: DeclareConditionContext): ErrorCondition = { + val conditionName = ctx.multipartIdentifier().getText + val conditionValue = Option(ctx.stringLit()).map(_.getText.replace("'", "")).getOrElse("45000") + + val sqlStateRegex = "^[A-Za-z0-9]{5}$".r + assert(sqlStateRegex.findFirstIn(conditionValue).isDefined) + + ErrorCondition(conditionName, conditionValue) + } + + override def visitDeclareHandler(ctx: DeclareHandlerContext): ErrorHandler = { + val conditions = visit(ctx.conditionValueList()).asInstanceOf[Seq[String]] + val handlerType = Option(ctx.EXIT()).map(_ => HandlerType.EXIT).getOrElse(HandlerType.CONTINUE) + + val body = Option(ctx.compoundBody()).map(visit).getOrElse { + val logicalPlan = visitChildren(ctx).asInstanceOf[LogicalPlan] + CompoundBody(Seq(SingleStatement(parsedPlan = logicalPlan))) + }.asInstanceOf[CompoundBody] + + ErrorHandler(conditions, body, handlerType) + } + override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = withOrigin(ctx) { Option(ctx.statement().asInstanceOf[ParserRuleContext]) .orElse(Option(ctx.setResetStatement().asInstanceOf[ParserRuleContext])) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingLogicalOperators.scala index 4a5259f09a8a3..70df497245838 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingLogicalOperators.scala @@ -17,6 +17,9 @@ package org.apache.spark.sql.catalyst.parser +import scala.collection.mutable + +import org.apache.spark.sql.catalyst.parser.HandlerType.HandlerType import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin, WithOrigin} @@ -54,10 +57,39 @@ case class SingleStatement(parsedPlan: LogicalPlan) * @param label Label set to CompoundBody by user or UUID otherwise. * It can be None in case when CompoundBody is not part of BeginEndCompoundBlock * for example when CompoundBody is inside loop or conditional block. + * @param handlers Collection of handlers defined in the compound body. + * @param conditions Map of Condition Name - Sql State values declared in the compound body. */ case class CompoundBody( collection: Seq[CompoundPlanStatement], - label: Option[String]) extends CompoundPlanStatement + label: Option[String] = None, + handlers: Seq[ErrorHandler] = Seq.empty, + conditions: mutable.HashMap[String, String] = mutable.HashMap()) extends CompoundPlanStatement + +/** + * Logical operator for an error condition. + * @param conditionName Name of the error condition. + * @param value SQLSTATE or Error Code. + */ +case class ErrorCondition( + conditionName: String, + value: String) extends CompoundPlanStatement + +object HandlerType extends Enumeration { + type HandlerType = Value + val EXIT, CONTINUE = Value +} + +/** + * Logical operator for an error condition. + * @param conditions Name of the error condition variable for which the handler is built. + * @param body CompoundBody of the handler. + * @param handlerType Type of the handler (CONTINUE or EXIT). + */ +case class ErrorHandler( + conditions: Seq[String], + body: CompoundBody, + handlerType: HandlerType) extends CompoundPlanStatement /** * Logical operator for IF ELSE statement. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/SqlScriptingErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/SqlScriptingErrors.scala index 61661b1d32f39..1683901ceb9ab 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/SqlScriptingErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/SqlScriptingErrors.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.errors import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.errors.QueryExecutionErrors.toSQLStmt import org.apache.spark.sql.exceptions.SqlScriptingException +import org.apache.spark.sql.internal.SQLConf /** * Object for grouping error messages thrown during parsing/interpreting phase @@ -43,6 +44,40 @@ private[sql] object SqlScriptingErrors { messageParameters = Map("endLabel" -> endLabel)) } + def sqlScriptingNotEnabled(origin: Origin): Throwable = { + new SqlScriptingException( + origin = origin, + errorClass = "UNSUPPORTED_FEATURE.SQL_SCRIPTING_NOT_ENABLED", + cause = null, + messageParameters = Map("sqlScriptingEnabled" -> SQLConf.SQL_SCRIPTING_ENABLED.key)) + } + + def duplicateHandlerForSameSqlState(origin: Origin, sqlState: String): Throwable = { + new SqlScriptingException( + origin = origin, + errorClass = "DUPLICATE_HANDLER_FOR_SAME_SQL_STATE", + cause = null, + messageParameters = Map("sqlState" -> sqlState)) + } + + def duplicateSqlStateForSameHandler(origin: Origin, sqlState: String): Throwable = { + new SqlScriptingException( + origin = origin, + errorClass = "DUPLICATE_SQL_STATE_FOR_SAME_HANDLER", + cause = null, + messageParameters = Map("sqlState" -> sqlState)) + } + + def duplicateConditionNameForDifferentSqlState( + origin: Origin, + conditionName: String): Throwable = { + new SqlScriptingException( + origin = origin, + errorClass = "DUPLICATE_CONDITION_NAME_FOR_DIFFERENT_SQL_STATE", + cause = null, + messageParameters = Map("conditionName" -> conditionName)) + } + def variableDeclarationNotAllowedInScope( origin: Origin, varName: String, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 6f2ebbc89cd1c..5c08c242d197b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -3355,6 +3355,14 @@ object SQLConf { .version("2.3.0") .fallbackConf(org.apache.spark.internal.config.STRING_REDACTION_PATTERN) + val SQL_SCRIPTING_ENABLED = + buildConf("spark.sql.scripting.enabled") + .doc("SQL Scripting feature is under development and its use should be done under this" + + "feature flag.") + .version("4.0.0") + .booleanConf + .createWithDefault(Utils.isTesting) + val CONCAT_BINARY_AS_STRING = buildConf("spark.sql.function.concatBinaryAsString") .doc("When this option is set to false and all inputs are binary, `functions.concat` returns " + "an output as binary. Otherwise, it returns as a string.") @@ -5533,6 +5541,8 @@ class SQLConf extends Serializable with Logging with SqlApiConf { def stringRedactionPattern: Option[Regex] = getConf(SQL_STRING_REDACTION_PATTERN) + def sqlScriptingEnabled: Boolean = getConf(SQL_SCRIPTING_ENABLED) + def sortBeforeRepartition: Boolean = getConf(SORT_BEFORE_REPARTITION) def topKSortFallbackThreshold: Int = getConf(TOP_K_SORT_FALLBACK_THRESHOLD) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingParserSuite.scala index 5fc3ade408bd9..56441389f199f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingParserSuite.scala @@ -21,6 +21,7 @@ import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.catalyst.plans.logical.CreateVariable import org.apache.spark.sql.exceptions.SqlScriptingException +import org.apache.spark.sql.internal.SQLConf class SqlScriptingParserSuite extends SparkFunSuite with SQLHelper { import CatalystSqlParser._ @@ -363,6 +364,81 @@ class SqlScriptingParserSuite extends SparkFunSuite with SQLHelper { assert(e.getMessage.contains("Syntax error")) } + test("declare condition: default sqlstate") { + val sqlScriptText = + """ + |BEGIN + | DECLARE test CONDITION; + |END""".stripMargin + val tree = parseScript(sqlScriptText) + assert(tree.conditions.size == 1) + assert(tree.conditions("test").equals("45000")) // Default SQLSTATE + } + + test("declare condition: custom sqlstate") { + val sqlScriptText = + """ + |BEGIN + | SELECT 1; + | DECLARE test CONDITION FOR '12000'; + |END""".stripMargin + val tree = parseScript(sqlScriptText) + assert(tree.conditions.size == 1) + assert(tree.conditions("test").equals("12000")) + } + + test("declare handler") { + val sqlScriptText = + """ + |BEGIN + | DECLARE CONTINUE HANDLER FOR test BEGIN SELECT 1; END; + |END""".stripMargin + val tree = parseScript(sqlScriptText) + assert(tree.handlers.length == 1) + assert(tree.handlers.head.isInstanceOf[ErrorHandler]) + } + + test("declare handler single statement") { + val sqlScriptText = + """ + |BEGIN + | DECLARE CONTINUE HANDLER FOR test SELECT 1; + |END""".stripMargin + val tree = parseScript(sqlScriptText) + assert(tree.handlers.length == 1) + assert(tree.handlers.head.isInstanceOf[ErrorHandler]) + } + + test("declare handler duplicate sqlState") { + val sqlScriptText = + """ + |BEGIN + | DECLARE CONTINUE HANDLER FOR test, test BEGIN SELECT 1; END; + |END""".stripMargin + checkError( + exception = intercept[SqlScriptingException] { + parseScript(sqlScriptText) + }, + errorClass = "DUPLICATE_SQL_STATE_FOR_SAME_HANDLER", + parameters = Map("sqlState" -> "test")) + } + + test("SQL Scripting not enabled") { + withSQLConf(SQLConf.SQL_SCRIPTING_ENABLED.key -> "false") { + val sqlScriptText = + """ + |BEGIN + | SELECT 1; + |END""".stripMargin + checkError( + exception = intercept[SqlScriptingException] { + parseScript(sqlScriptText) + }, + errorClass = "UNSUPPORTED_FEATURE.SQL_SCRIPTING_NOT_ENABLED", + parameters = Map("sqlScriptingEnabled" -> SQLConf.SQL_SCRIPTING_ENABLED.key)) + } + } + test("if") { val sqlScriptText = """ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index d64623a744fe4..15fe6262fba26 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -40,7 +40,9 @@ import org.apache.spark.sql.catalyst._ import org.apache.spark.sql.catalyst.analysis.{NameParameterizedQuery, PosParameterizedQuery, UnresolvedRelation} import org.apache.spark.sql.catalyst.encoders._ import org.apache.spark.sql.catalyst.expressions.AttributeReference +import org.apache.spark.sql.catalyst.parser.{CompoundBody, SingleStatement} import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Range} +import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.connector.ExternalCommandRunner @@ -632,6 +634,11 @@ class SparkSession private( | Everything else | * ----------------- */ + private def executeScript(compoundBody: CompoundBody): Iterator[Array[Row]] = { + val interpreter = sessionState.sqlScriptingInterpreter + interpreter.execute(compoundBody) + } + /** * Executes a SQL query substituting positional parameters by the given arguments, * returning the result as a `DataFrame`. @@ -650,14 +657,27 @@ class SparkSession private( private[sql] def sql(sqlText: String, args: Array[_], tracker: QueryPlanningTracker): DataFrame = withActive { val plan = tracker.measurePhase(QueryPlanningTracker.PARSING) { - val parsedPlan = sessionState.sqlParser.parsePlan(sqlText) - if (args.nonEmpty) { - PosParameterizedQuery(parsedPlan, args.map(lit(_).expr).toImmutableArraySeq) - } else { - parsedPlan + val parsedPlan = sessionState.sqlParser.parseScript(sqlText) + parsedPlan match { + case CompoundBody(Seq(singleStmtPlan: SingleStatement), label, _, _) if args.nonEmpty => + CompoundBody(Seq(SingleStatement( + PosParameterizedQuery( + singleStmtPlan.parsedPlan, args.map(lit(_).expr).toImmutableArraySeq))), label) + case p => + assert(args.isEmpty, "Named parameters are not supported for batch queries") + p } } - Dataset.ofRows(self, plan, tracker) + + plan match { + case CompoundBody(Seq(singleStmtPlan: SingleStatement), _, _, _) => + Dataset.ofRows(self, singleStmtPlan.parsedPlan, tracker) + case _ => + // execute the plan directly if it is not a single statement + val lastRow = executeScript(plan).foldLeft(Array.empty[Row])((_, next) => next) + val attributes = DataTypeUtils.toAttributes(lastRow.head.schema) + Dataset.ofRows(self, LocalRelation.fromExternalRows(attributes, lastRow.toIndexedSeq)) + } } /** @@ -703,14 +723,27 @@ class SparkSession private( tracker: QueryPlanningTracker): DataFrame = withActive { val plan = tracker.measurePhase(QueryPlanningTracker.PARSING) { - val parsedPlan = sessionState.sqlParser.parsePlan(sqlText) - if (args.nonEmpty) { - NameParameterizedQuery(parsedPlan, args.transform((_, v) => lit(v).expr)) - } else { - parsedPlan + val parsedPlan = sessionState.sqlParser.parseScript(sqlText) + parsedPlan match { + case CompoundBody(Seq(singleStmtPlan: SingleStatement), label, _, _) if args.nonEmpty => + CompoundBody(Seq(SingleStatement( + NameParameterizedQuery( + singleStmtPlan.parsedPlan, args.transform((_, v) => lit(v).expr)))), label) + case p => + assert(args.isEmpty, "Positional parameters are not supported for batch queries") + p } } - Dataset.ofRows(self, plan, tracker) + + plan match { + case CompoundBody(Seq(singleStmtPlan: SingleStatement), _, _, _) => + Dataset.ofRows(self, singleStmtPlan.parsedPlan, tracker) + case _ => + // execute the plan directly if it is not a single statement + val lastRow = executeScript(plan).foldLeft(Array.empty[Row])((_, next) => next) + val attributes = DataTypeUtils.toAttributes(lastRow.head.schema) + Dataset.ofRows(self, LocalRelation.fromExternalRows(attributes, lastRow.toIndexedSeq)) + } } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala index 677dba0082575..93a00521eef86 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala @@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.{ColumnarRule, SparkPlan} +import org.apache.spark.sql.scripting.SqlScriptingInterpreter /** * :: Experimental :: @@ -110,6 +111,7 @@ class SparkSessionExtensions { type CheckRuleBuilder = SparkSession => LogicalPlan => Unit type StrategyBuilder = SparkSession => Strategy type ParserBuilder = (SparkSession, ParserInterface) => ParserInterface + type InterpreterBuilder = (SparkSession, SqlScriptingInterpreter) => SqlScriptingInterpreter type FunctionDescription = (FunctionIdentifier, ExpressionInfo, FunctionBuilder) type TableFunctionDescription = (FunctionIdentifier, ExpressionInfo, TableFunctionBuilder) type ColumnarRuleBuilder = SparkSession => ColumnarRule @@ -330,6 +332,16 @@ class SparkSessionExtensions { } } + private[this] val interpreterBuilders = mutable.Buffer.empty[InterpreterBuilder] + + private[sql] def buildInterpreter( + session: SparkSession, + initial: SqlScriptingInterpreter): SqlScriptingInterpreter = { + interpreterBuilders.foldLeft(initial) { (interpreter, builder) => + builder(session, interpreter) + } + } + /** * Inject a custom parser into the [[SparkSession]]. Note that the builder is passed a session * and an initial parser. The latter allows for a user to create a partial parser and to delegate diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala index 99287bddb5104..24852e769df36 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala @@ -37,6 +37,7 @@ import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.v2.{TableCapabilityCheck, V2SessionCatalog} import org.apache.spark.sql.execution.streaming.ResolveWriteToStream import org.apache.spark.sql.expressions.UserDefinedAggregateFunction +import org.apache.spark.sql.scripting.SqlScriptingInterpreter import org.apache.spark.sql.streaming.StreamingQueryManager import org.apache.spark.sql.util.ExecutionListenerManager @@ -146,6 +147,13 @@ abstract class BaseSessionStateBuilder( extensions.buildParser(session, new SparkSqlParser()) } + /** + * Script interpreter that produces execution plan and executes SQL scripts. + */ + protected lazy val scriptingInterpreter: SqlScriptingInterpreter = { + extensions.buildInterpreter(session, SqlScriptingInterpreter(session)) + } + /** * ResourceLoader that is used to load function resources and jars. */ @@ -395,6 +403,7 @@ abstract class BaseSessionStateBuilder( dataSourceRegistration, () => catalog, sqlParser, + scriptingInterpreter, () => analyzer, () => optimizer, planner, diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala index bc6710e6cbdb8..4548da4ed2842 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala @@ -36,6 +36,7 @@ import org.apache.spark.sql.connector.catalog.CatalogManager import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.AdaptiveRulesHolder import org.apache.spark.sql.execution.datasources.DataSourceManager +import org.apache.spark.sql.scripting.SqlScriptingInterpreter import org.apache.spark.sql.streaming.StreamingQueryManager import org.apache.spark.sql.util.ExecutionListenerManager import org.apache.spark.util.{DependencyUtils, Utils} @@ -80,6 +81,7 @@ private[sql] class SessionState( val dataSourceRegistration: DataSourceRegistration, catalogBuilder: () => SessionCatalog, val sqlParser: ParserInterface, + val sqlScriptingInterpreter: SqlScriptingInterpreter, analyzerBuilder: () => Analyzer, optimizerBuilder: () => Optimizer, val planner: SparkPlanner, diff --git a/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNode.scala b/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNode.scala index a21392da44743..d4a31aac8d793 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNode.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNode.scala @@ -17,10 +17,12 @@ package org.apache.spark.sql.scripting -import org.apache.spark.SparkException +import scala.collection.mutable + +import org.apache.spark.{SparkException, SparkThrowable} import org.apache.spark.internal.Logging -import org.apache.spark.sql.{Dataset, SparkSession} -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.{Dataset, Row, SparkSession} +import org.apache.spark.sql.catalyst.plans.logical.{DropVariable, LogicalPlan} import org.apache.spark.sql.catalyst.trees.{Origin, WithOrigin} import org.apache.spark.sql.errors.SqlScriptingErrors import org.apache.spark.sql.types.BooleanType @@ -45,7 +47,29 @@ sealed trait CompoundStatementExec extends Logging { /** * Leaf node in the execution tree. */ -trait LeafStatementExec extends CompoundStatementExec +trait LeafStatementExec extends CompoundStatementExec { + + /** Whether an error was raised during the execution of this statement. */ + var raisedError: Boolean = false + + /** Error state of the statement. */ + var errorState: Option[String] = None + + /** Throwable to rethrow after the statement execution if the error is not handled. */ + var rethrow: Option[Throwable] = None + + /** + * Execute the statement. + * @param session Spark session. + */ + def execute(session: SparkSession): Unit + + override def reset(): Unit = { + raisedError = false + errorState = None + rethrow = None + } +} /** * Non-leaf node in the execution tree. It is an iterator over executable child nodes. @@ -72,9 +96,6 @@ trait NonLeafStatementExec extends CompoundStatementExec { session: SparkSession, statement: LeafStatementExec): Boolean = statement match { case statement: SingleStatementExec => - assert(!statement.isExecuted) - statement.isExecuted = true - // DataFrame evaluates to True if it is single row, single column // of boolean type with value True. val df = Dataset.ofRows(session, statement.parsedPlan) @@ -109,18 +130,19 @@ trait NonLeafStatementExec extends CompoundStatementExec { * Whether the statement originates from the SQL script or it is created during the * interpretation. Example: DropVariable statements are automatically created at the end of each * compound. + * @param shouldCollectResult + * Whether we should collect result after statement execution. Example: results from conditions + * in if-else or loops should not be collected. */ class SingleStatementExec( var parsedPlan: LogicalPlan, override val origin: Origin, - override val isInternal: Boolean) + override val isInternal: Boolean = false, + val shouldCollectResult: Boolean = false) extends LeafStatementExec with WithOrigin { - /** - * Whether this statement has been executed during the interpretation phase. - * Example: Statements in conditions of If/Else, While, etc. - */ - var isExecuted = false + /** Data returned after execution. */ + var result: Option[Array[Row]] = None /** * Get the SQL query text corresponding to this statement. @@ -132,20 +154,138 @@ class SingleStatementExec( origin.sqlText.get.substring(origin.startIndex.get, origin.stopIndex.get + 1) } - override def reset(): Unit = isExecuted = false + override def reset(): Unit = { + super.reset() + result = None + } + + override def execute(session: SparkSession): Unit = { + try { + val rows = Some(Dataset.ofRows(session, parsedPlan).collect()) + if (shouldCollectResult) { + result = rows + } + } catch { + case e: SparkThrowable => + raisedError = true + errorState = Some(e.getSqlState) + e match { + case throwable: Throwable => + rethrow = Some(throwable) + case _ => + } + case throwable: Throwable => + raisedError = true + errorState = Some("SQLEXCEPTION") + rethrow = Some(throwable) + } + } } /** - * Abstract class for all statements that contain nested statements. - * Implements recursive iterator logic over all child execution nodes. - * @param collection - * Collection of child execution nodes. + * Executable node for CompoundBody. + * @param statements + * Executable nodes for nested statements within the CompoundBody. + * @param session + * Spark session. */ -abstract class CompoundNestedStatementIteratorExec(collection: Seq[CompoundStatementExec]) +class CompoundBodyExec( + statements: Seq[CompoundStatementExec], + session: SparkSession, + label: Option[String] = None, + conditionHandlerMap: mutable.HashMap[String, ErrorHandlerExec] = mutable.HashMap()) extends NonLeafStatementExec { - private var localIterator = collection.iterator - private var curr = if (localIterator.hasNext) Some(localIterator.next()) else None + /** + * Get handler to handle error given by condition. + * @param condition SqlState of the error raised during statement execution. + * @return Corresponding error handler executable node. + */ + private def getHandler(condition: String): Option[ErrorHandlerExec] = { + conditionHandlerMap.get(condition) + .orElse{ + conditionHandlerMap.get("NOT FOUND") match { + // If NOT FOUND handler is defined, use it only for errors with class '02'. + case Some(handler) if condition.startsWith("02") => Some(handler) + case _ => None + }} + .orElse{conditionHandlerMap.get("SQLEXCEPTION")} + } + + /** + * Handle error raised during the execution of the statement. + * @param statement statement that possibly raised the error + * @return pass through the statement + */ + private def handleError(statement: LeafStatementExec): LeafStatementExec = { + if (statement.raisedError) { + getHandler(statement.errorState.get).foreach { handler => + statement.reset() // Clear all flags and result + handler.reset() + returnHere = curr + curr = Some(handler.body) + } + } + statement + } + + /** + * Drop variables declared in this CompoundBody. + */ + private def cleanup(): Unit = { + // Filter out internal DropVariable statements and execute them. + statements.filter( + dropVar => dropVar.isInstanceOf[SingleStatementExec] + && dropVar.asInstanceOf[SingleStatementExec].parsedPlan.isInstanceOf[DropVariable] + && dropVar.isInternal) + .foreach(_.asInstanceOf[SingleStatementExec].execute(session)) + } + + /** + * Check if the leave statement was used, if it is not used stop iterating surrounding + * [[CompoundBodyExec]] and move iterator forward. If the label of the block matches the label of + * the leave statement, mark the leave statement as used. + * @param leave leave statement + * @return pass through the leave statement + */ + private def handleLeave(leave: LeaveStatementExec): LeaveStatementExec = { + if (!leave.used) { + // Hard stop the iteration of the current begin/end block. + stopIteration = true + // Cleanup variables declared in the current block. + cleanup() + // If label of the block matches the label of the leave statement, + // mark the leave statement as used. label can be None in case of a + // CompoundBody inside loop or if/else structure. In such cases, + // loop will have its own label to be matched by leave statement. + if (label.isDefined) { + leave.used = label.get.equals(leave.label) + } else { + leave.used = false + } + } + curr = if (localIterator.hasNext) Some(localIterator.next()) else None + leave + } + + private var localIterator: Iterator[CompoundStatementExec] = statements.iterator + private var curr: Option[CompoundStatementExec] = + if (localIterator.hasNext) Some(localIterator.next()) else None + + // Flag to stop the iteration of the current begin/end block. + // It is set to true when non-consumed leave statement is encountered. + private var stopIteration: Boolean = false + + // Statement to return to after handling the error with continue handler. + private var returnHere: Option[CompoundStatementExec] = None + + def getTreeIterator: Iterator[CompoundStatementExec] = treeIterator + + override def reset(): Unit = { + statements.foreach(_.reset()) + localIterator = statements.iterator + curr = if (localIterator.hasNext) Some(localIterator.next()) else None + } private lazy val treeIterator: Iterator[CompoundStatementExec] = new Iterator[CompoundStatementExec] { @@ -157,7 +297,7 @@ abstract class CompoundNestedStatementIteratorExec(collection: Seq[CompoundState case _ => throw SparkException.internalError( "Unknown statement type encountered during SQL script interpretation.") } - localIterator.hasNext || childHasNext + (localIterator.hasNext || childHasNext || returnHere.isDefined) && !stopIteration } @scala.annotation.tailrec @@ -165,14 +305,31 @@ abstract class CompoundNestedStatementIteratorExec(collection: Seq[CompoundState curr match { case None => throw SparkException.internalError( "No more elements to iterate through in the current SQL compound statement.") + case Some(leave: LeaveStatementExec) => + handleLeave(leave) case Some(statement: LeafStatementExec) => + statement.execute(session) // Execute the leaf statement curr = if (localIterator.hasNext) Some(localIterator.next()) else None - statement + handleError(statement) // Handle error if raised case Some(body: NonLeafStatementExec) => if (body.getTreeIterator.hasNext) { - body.getTreeIterator.next() + val statement = body.getTreeIterator.next() // Get next statement from the child node + statement match { + case leave: LeaveStatementExec => + handleLeave(leave) + case leafStatement: LeafStatementExec => + // This check is done to handle error in surrounding begin/end block + // if it was not handled in the nested block. + handleError(leafStatement) + case nonLeafStatement: NonLeafStatementExec => nonLeafStatement + } } else { - curr = if (localIterator.hasNext) Some(localIterator.next()) else None + if (returnHere.isDefined) { + curr = returnHere + returnHere = None + } else { + curr = if (localIterator.hasNext) Some(localIterator.next()) else None + } next() } case _ => throw SparkException.internalError( @@ -180,23 +337,28 @@ abstract class CompoundNestedStatementIteratorExec(collection: Seq[CompoundState } } } +} - override def getTreeIterator: Iterator[CompoundStatementExec] = treeIterator +class ErrorHandlerExec(val body: CompoundBodyExec) extends NonLeafStatementExec { - override def reset(): Unit = { - collection.foreach(_.reset()) - localIterator = collection.iterator - curr = if (localIterator.hasNext) Some(localIterator.next()) else None - } + override def getTreeIterator: Iterator[CompoundStatementExec] = body.getTreeIterator + + override def reset(): Unit = body.reset() } /** - * Executable node for CompoundBody. - * @param statements - * Executable nodes for nested statements within the CompoundBody. + * Executable node for Leave statement. + * @param label + * Label of the [[CompoundBodyExec]] that should be exited. */ -class CompoundBodyExec(statements: Seq[CompoundStatementExec]) - extends CompoundNestedStatementIteratorExec(statements) +class LeaveStatementExec(val label: String) extends LeafStatementExec { + + var used: Boolean = false + + override def execute(session: SparkSession): Unit = () + + override def reset(): Unit = used = false +} /** * Executable node for IfElseStatement. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingInterpreter.scala b/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingInterpreter.scala index 08b4f97286280..9e5f80fb784db 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingInterpreter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingInterpreter.scala @@ -17,16 +17,20 @@ package org.apache.spark.sql.scripting -import org.apache.spark.sql.SparkSession +import scala.collection.mutable + +import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier -import org.apache.spark.sql.catalyst.parser.{CompoundBody, CompoundPlanStatement, IfElseStatement, SingleStatement, WhileStatement} +import org.apache.spark.sql.catalyst.parser.{CompoundBody, CompoundPlanStatement, HandlerType, IfElseStatement, SingleStatement, WhileStatement} import org.apache.spark.sql.catalyst.plans.logical.{CreateVariable, DropVariable, LogicalPlan} -import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin} +import org.apache.spark.sql.errors.SqlScriptingErrors + /** * SQL scripting interpreter - builds SQL script execution plan. */ -case class SqlScriptingInterpreter() { +case class SqlScriptingInterpreter(session: SparkSession) { /** * Build execution plan and return statements that need to be executed, @@ -34,15 +38,11 @@ case class SqlScriptingInterpreter() { * * @param compound * CompoundBody for which to build the plan. - * @param session - * Spark session that SQL script is executed within. * @return * Iterator through collection of statements to be executed. */ - def buildExecutionPlan( - compound: CompoundBody, - session: SparkSession): Iterator[CompoundStatementExec] = { - transformTreeIntoExecutable(compound, session).asInstanceOf[CompoundBodyExec].getTreeIterator + private def buildExecutionPlan(compound: CompoundBody): Iterator[CompoundStatementExec] = { + transformTreeIntoExecutable(compound).asInstanceOf[CompoundBodyExec].getTreeIterator } /** @@ -58,50 +58,126 @@ case class SqlScriptingInterpreter() { case _ => None } + /** + * Transform [[CompoundBody]] into [[CompoundBodyExec]]. + * @param compoundBody + * CompoundBody to be transformed into CompoundBodyExec. + * @param isExitHandler + * Flag to indicate if the body is an exit handler body to add leave statement at the end. + * @param exitHandlerLabel + * If body is an exit handler body, this is the label of surrounding CompoundBody + * that should be exited. + * @return + * Executable version of the CompoundBody . + */ + private def transformBodyIntoExec( + compoundBody: CompoundBody, + isExitHandler: Boolean = false, + exitHandlerLabel: String = ""): CompoundBodyExec = { + val variables = compoundBody.collection.flatMap { + case st: SingleStatement => getDeclareVarNameFromPlan(st.parsedPlan) + case _ => None + } + val dropVariables = variables + .map(varName => DropVariable(varName, ifExists = true)) + .map(new SingleStatementExec(_, Origin(), isInternal = true)) + .reverse + + // Create a map of conditions (SqlStates) to their respective handlers. + val conditionHandlerMap = mutable.HashMap[String, ErrorHandlerExec]() + compoundBody.handlers.foreach(handler => { + val handlerBodyExec = + transformBodyIntoExec(handler.body, + handler.handlerType == HandlerType.EXIT, + compoundBody.label.get) + + // Execution node of handler. + val handlerExec = new ErrorHandlerExec(handlerBodyExec) + + // For each condition handler is defined for, add corresponding key value pair + // to the conditionHandlerMap. + handler.conditions.foreach(condition => { + // Condition can either be the key in conditions map or SqlState. + val conditionValue = compoundBody.conditions.getOrElse(condition, condition) + if (conditionHandlerMap.contains(conditionValue)) { + throw SqlScriptingErrors.duplicateHandlerForSameSqlState( + CurrentOrigin.get, conditionValue) + } else { + conditionHandlerMap.put(conditionValue, handlerExec) + } + }) + }) + + if (isExitHandler) { + // Create leave statement to exit the surrounding CompoundBody after handler execution. + val leave = new LeaveStatementExec(exitHandlerLabel) + val statements = compoundBody.collection.map(st => transformTreeIntoExecutable(st)) ++ + dropVariables :+ leave + + return new CompoundBodyExec(statements, session, compoundBody.label, conditionHandlerMap) + } + + new CompoundBodyExec( + compoundBody.collection.map(st => transformTreeIntoExecutable(st)) ++ dropVariables, + session, + compoundBody.label, + conditionHandlerMap) + } + /** * Transform the parsed tree to the executable node. * * @param node * Root node of the parsed tree. - * @param session - * Spark session that SQL script is executed within. * @return * Executable statement. */ - private def transformTreeIntoExecutable( - node: CompoundPlanStatement, session: SparkSession): CompoundStatementExec = + private def transformTreeIntoExecutable(node: CompoundPlanStatement): CompoundStatementExec = node match { case body: CompoundBody => // TODO [SPARK-48530]: Current logic doesn't support scoped variables and shadowing. - val variables = body.collection.flatMap { - case st: SingleStatement => getDeclareVarNameFromPlan(st.parsedPlan) - case _ => None - } - val dropVariables = variables - .map(varName => DropVariable(varName, ifExists = true)) - .map(new SingleStatementExec(_, Origin(), isInternal = true)) - .reverse - new CompoundBodyExec( - body.collection.map(st => transformTreeIntoExecutable(st, session)) ++ dropVariables) + transformBodyIntoExec(body) case IfElseStatement(conditions, conditionalBodies, elseBody) => val conditionsExec = conditions.map(condition => new SingleStatementExec(condition.parsedPlan, condition.origin, isInternal = false)) val conditionalBodiesExec = conditionalBodies.map(body => - transformTreeIntoExecutable(body, session).asInstanceOf[CompoundBodyExec]) + transformTreeIntoExecutable(body).asInstanceOf[CompoundBodyExec]) val unconditionalBodiesExec = elseBody.map(body => - transformTreeIntoExecutable(body, session).asInstanceOf[CompoundBodyExec]) + transformTreeIntoExecutable(body).asInstanceOf[CompoundBodyExec]) new IfElseStatementExec( conditionsExec, conditionalBodiesExec, unconditionalBodiesExec, session) case WhileStatement(condition, body, _) => val conditionExec = new SingleStatementExec(condition.parsedPlan, condition.origin, isInternal = false) val bodyExec = - transformTreeIntoExecutable(body, session).asInstanceOf[CompoundBodyExec] + transformTreeIntoExecutable(body).asInstanceOf[CompoundBodyExec] new WhileStatementExec(conditionExec, bodyExec, session) case sparkStatement: SingleStatement => new SingleStatementExec( sparkStatement.parsedPlan, sparkStatement.origin, - isInternal = false) + shouldCollectResult = true) + case _ => + throw new UnsupportedOperationException( + s"Unsupported operation in the execution plan.") + } + + def execute(compoundBody: CompoundBody): Iterator[Array[Row]] = { + val executionPlan = buildExecutionPlan(compoundBody) + executionPlan.flatMap { + case statement: SingleStatementExec if statement.raisedError => + val sqlState = statement.errorState.getOrElse(throw statement.rethrow.get) + + // SQLWARNING and NOT FOUND are not considered as errors. + if (!sqlState.startsWith("01") && !sqlState.startsWith("02")) { + // Throw the error for SQLEXCEPTION. + throw statement.rethrow.get + } + + // Return empty result set for SQLWARNING and NOT FOUND. + None + case statement: SingleStatementExec if statement.shouldCollectResult => statement.result + case _ => None } + } } diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/literals.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/literals.sql.out index 570cfb73444e5..738d27f0873e1 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/literals.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/literals.sql.out @@ -239,7 +239,7 @@ org.apache.spark.sql.catalyst.parser.ParseException "errorClass" : "PARSE_SYNTAX_ERROR", "sqlState" : "42601", "messageParameters" : { - "error" : "'.'", + "error" : "end of input", "hint" : "" } } diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out index 570cfb73444e5..738d27f0873e1 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out @@ -239,7 +239,7 @@ org.apache.spark.sql.catalyst.parser.ParseException "errorClass" : "PARSE_SYNTAX_ERROR", "sqlState" : "42601", "messageParameters" : { - "error" : "'.'", + "error" : "end of input", "hint" : "" } } diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/keywords.sql.out index 5735e5eef68e7..c5fb9750fd23b 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/keywords.sql.out @@ -58,8 +58,10 @@ COMPACTIONS false COMPENSATION false COMPUTE false CONCATENATE false +CONDITION true CONSTRAINT true CONTAINS false +CONTINUE false COST false CREATE true CROSS true @@ -111,6 +113,7 @@ EXCHANGE false EXCLUDE false EXECUTE true EXISTS false +EXIT false EXPLAIN false EXPORT false EXTENDED false @@ -128,6 +131,7 @@ FOR true FOREIGN true FORMAT false FORMATTED false +FOUND false FROM true FULL true FUNCTION false @@ -137,6 +141,7 @@ GLOBAL false GRANT true GROUP true GROUPING false +HANDLER false HAVING true HOUR false HOURS false @@ -288,6 +293,7 @@ SORTED false SOURCE false SPECIFIC false SQL true +SQLEXCEPTION false START false STATISTICS false STORED false @@ -378,6 +384,7 @@ CHECK COLLATE COLLATION COLUMN +CONDITION CONSTRAINT CREATE CROSS diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out index 4e4c70cc333ba..672d7f1567e87 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out @@ -269,7 +269,7 @@ org.apache.spark.sql.catalyst.parser.ParseException "errorClass" : "PARSE_SYNTAX_ERROR", "sqlState" : "42601", "messageParameters" : { - "error" : "'.'", + "error" : "end of input", "hint" : "" } } diff --git a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out index ca48e851e717c..f669298420272 100644 --- a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out @@ -58,8 +58,10 @@ COMPACTIONS false COMPENSATION false COMPUTE false CONCATENATE false +CONDITION true CONSTRAINT false CONTAINS false +CONTINUE false COST false CREATE false CROSS false @@ -111,6 +113,7 @@ EXCHANGE false EXCLUDE false EXECUTE false EXISTS false +EXIT false EXPLAIN false EXPORT false EXTENDED false @@ -128,6 +131,7 @@ FOR false FOREIGN false FORMAT false FORMATTED false +FOUND false FROM false FULL false FUNCTION false @@ -137,6 +141,7 @@ GLOBAL false GRANT false GROUP false GROUPING false +HANDLER false HAVING false HOUR false HOURS false @@ -288,6 +293,7 @@ SORTED false SOURCE false SPECIFIC false SQL false +SQLEXCEPTION false START false STATISTICS false STORED false @@ -366,4 +372,4 @@ SELECT keyword from SQL_KEYWORDS() WHERE reserved -- !query schema struct -- !query output - +CONDITION diff --git a/sql/core/src/test/resources/sql-tests/results/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/literals.sql.out index 4e4c70cc333ba..672d7f1567e87 100644 --- a/sql/core/src/test/resources/sql-tests/results/literals.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/literals.sql.out @@ -269,7 +269,7 @@ org.apache.spark.sql.catalyst.parser.ParseException "errorClass" : "PARSE_SYNTAX_ERROR", "sqlState" : "42601", "messageParameters" : { - "error" : "'.'", + "error" : "end of input", "hint" : "" } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNodeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNodeSuite.scala index 5c36f9e19e6d9..b86927f349550 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNodeSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNodeSuite.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.scripting +import scala.collection.mutable + import org.apache.spark.SparkFunSuite import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Literal} @@ -33,8 +35,14 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi // Helpers case class TestLeafStatement(testVal: String) extends LeafStatementExec { override def reset(): Unit = () + + override def execute(session: SparkSession): Unit = () } + case class TestBody(statements: Seq[CompoundStatementExec]) + extends CompoundBodyExec(statements, null, None, mutable.HashMap()) + + case class TestSparkStatementWithPlan(testVal: String) case class TestIfElseCondition(condVal: Boolean, description: String) extends SingleStatementExec( parsedPlan = Project(Seq(Alias(Literal(condVal), description)()), OneRowRelation()), @@ -82,13 +90,13 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi // Tests test("test body - single statement") { - val iter = new CompoundBodyExec(Seq(TestLeafStatement("one"))).getTreeIterator + val iter = TestBody(Seq(TestLeafStatement("one"))).getTreeIterator val statements = iter.map(extractStatementValue).toSeq assert(statements === Seq("one")) } test("test body - no nesting") { - val iter = new CompoundBodyExec( + val iter = TestBody( Seq( TestLeafStatement("one"), TestLeafStatement("two"), @@ -99,26 +107,26 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("test body - nesting") { - val iter = new CompoundBodyExec( + val iter = TestBody( Seq( - new CompoundBodyExec(Seq(TestLeafStatement("one"), TestLeafStatement("two"))), + TestBody(Seq(TestLeafStatement("one"), TestLeafStatement("two"))), TestLeafStatement("three"), - new CompoundBodyExec(Seq(TestLeafStatement("four"), TestLeafStatement("five"))))) + TestBody(Seq(TestLeafStatement("four"), TestLeafStatement("five"))))) .getTreeIterator val statements = iter.map(extractStatementValue).toSeq assert(statements === Seq("one", "two", "three", "four", "five")) } test("if else - enter body of the IF clause") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( new IfElseStatementExec( conditions = Seq( TestIfElseCondition(condVal = true, description = "con1") ), conditionalBodies = Seq( - new CompoundBodyExec(Seq(TestLeafStatement("body1"))) + TestBody(Seq(TestLeafStatement("body1"))) ), - elseBody = Some(new CompoundBodyExec(Seq(TestLeafStatement("body2")))), + elseBody = Some(TestBody(Seq(TestLeafStatement("body2")))), session = spark ) )).getTreeIterator @@ -127,15 +135,15 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("if else - enter body of the ELSE clause") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( new IfElseStatementExec( conditions = Seq( TestIfElseCondition(condVal = false, description = "con1") ), conditionalBodies = Seq( - new CompoundBodyExec(Seq(TestLeafStatement("body1"))) + TestBody(Seq(TestLeafStatement("body1"))) ), - elseBody = Some(new CompoundBodyExec(Seq(TestLeafStatement("body2")))), + elseBody = Some(TestBody(Seq(TestLeafStatement("body2")))), session = spark ) )).getTreeIterator @@ -144,17 +152,17 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("if else if - enter body of the IF clause") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( new IfElseStatementExec( conditions = Seq( TestIfElseCondition(condVal = true, description = "con1"), TestIfElseCondition(condVal = false, description = "con2") ), conditionalBodies = Seq( - new CompoundBodyExec(Seq(TestLeafStatement("body1"))), - new CompoundBodyExec(Seq(TestLeafStatement("body2"))) + TestBody(Seq(TestLeafStatement("body1"))), + TestBody(Seq(TestLeafStatement("body2"))) ), - elseBody = Some(new CompoundBodyExec(Seq(TestLeafStatement("body3")))), + elseBody = Some(TestBody(Seq(TestLeafStatement("body3")))), session = spark ) )).getTreeIterator @@ -163,17 +171,17 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("if else if - enter body of the ELSE IF clause") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( new IfElseStatementExec( conditions = Seq( TestIfElseCondition(condVal = false, description = "con1"), TestIfElseCondition(condVal = true, description = "con2") ), conditionalBodies = Seq( - new CompoundBodyExec(Seq(TestLeafStatement("body1"))), - new CompoundBodyExec(Seq(TestLeafStatement("body2"))) + TestBody(Seq(TestLeafStatement("body1"))), + TestBody(Seq(TestLeafStatement("body2"))) ), - elseBody = Some(new CompoundBodyExec(Seq(TestLeafStatement("body3")))), + elseBody = Some(TestBody(Seq(TestLeafStatement("body3")))), session = spark ) )).getTreeIterator @@ -182,7 +190,7 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("if else if - enter body of the second ELSE IF clause") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( new IfElseStatementExec( conditions = Seq( TestIfElseCondition(condVal = false, description = "con1"), @@ -190,11 +198,11 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi TestIfElseCondition(condVal = true, description = "con3") ), conditionalBodies = Seq( - new CompoundBodyExec(Seq(TestLeafStatement("body1"))), - new CompoundBodyExec(Seq(TestLeafStatement("body2"))), - new CompoundBodyExec(Seq(TestLeafStatement("body3"))) + TestBody(Seq(TestLeafStatement("body1"))), + TestBody(Seq(TestLeafStatement("body2"))), + TestBody(Seq(TestLeafStatement("body3"))) ), - elseBody = Some(new CompoundBodyExec(Seq(TestLeafStatement("body4")))), + elseBody = Some(TestBody(Seq(TestLeafStatement("body4")))), session = spark ) )).getTreeIterator @@ -203,17 +211,17 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("if else if - enter body of the ELSE clause") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( new IfElseStatementExec( conditions = Seq( TestIfElseCondition(condVal = false, description = "con1"), TestIfElseCondition(condVal = false, description = "con2") ), conditionalBodies = Seq( - new CompoundBodyExec(Seq(TestLeafStatement("body1"))), - new CompoundBodyExec(Seq(TestLeafStatement("body2"))) + TestBody(Seq(TestLeafStatement("body1"))), + TestBody(Seq(TestLeafStatement("body2"))) ), - elseBody = Some(new CompoundBodyExec(Seq(TestLeafStatement("body3")))), + elseBody = Some(TestBody(Seq(TestLeafStatement("body3")))), session = spark ) )).getTreeIterator @@ -222,15 +230,15 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("if else if - without else (successful check)") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( new IfElseStatementExec( conditions = Seq( TestIfElseCondition(condVal = false, description = "con1"), TestIfElseCondition(condVal = true, description = "con2") ), conditionalBodies = Seq( - new CompoundBodyExec(Seq(TestLeafStatement("body1"))), - new CompoundBodyExec(Seq(TestLeafStatement("body2"))) + TestBody(Seq(TestLeafStatement("body1"))), + TestBody(Seq(TestLeafStatement("body2"))) ), elseBody = None, session = spark @@ -241,15 +249,15 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("if else if - without else (unsuccessful checks)") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( new IfElseStatementExec( conditions = Seq( TestIfElseCondition(condVal = false, description = "con1"), TestIfElseCondition(condVal = false, description = "con2") ), conditionalBodies = Seq( - new CompoundBodyExec(Seq(TestLeafStatement("body1"))), - new CompoundBodyExec(Seq(TestLeafStatement("body2"))) + TestBody(Seq(TestLeafStatement("body1"))), + TestBody(Seq(TestLeafStatement("body2"))) ), elseBody = None, session = spark @@ -260,10 +268,10 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("while - doesn't enter body") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( TestWhile( condition = TestWhileCondition(condVal = true, reps = 0, description = "con1"), - body = new CompoundBodyExec(Seq(TestLeafStatement("body1"))) + body = TestBody(Seq(TestLeafStatement("body1"))) ) )).getTreeIterator val statements = iter.map(extractStatementValue).toSeq @@ -271,10 +279,10 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("while - enters body once") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( TestWhile( condition = TestWhileCondition(condVal = true, reps = 1, description = "con1"), - body = new CompoundBodyExec(Seq(TestLeafStatement("body1"))) + body = TestBody(Seq(TestLeafStatement("body1"))) ) )).getTreeIterator val statements = iter.map(extractStatementValue).toSeq @@ -282,10 +290,10 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("while - enters body with multiple statements multiple times") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( TestWhile( condition = TestWhileCondition(condVal = true, reps = 2, description = "con1"), - body = new CompoundBodyExec(Seq( + body = TestBody(Seq( TestLeafStatement("statement1"), TestLeafStatement("statement2"))) ) @@ -296,13 +304,13 @@ class SqlScriptingExecutionNodeSuite extends SparkFunSuite with SharedSparkSessi } test("nested while - 2 times outer 2 times inner") { - val iter = new CompoundBodyExec(Seq( + val iter = TestBody(Seq( TestWhile( condition = TestWhileCondition(condVal = true, reps = 2, description = "con1"), - body = new CompoundBodyExec(Seq( + body = TestBody(Seq( TestWhile( condition = TestWhileCondition(condVal = true, reps = 2, description = "con2"), - body = new CompoundBodyExec(Seq(TestLeafStatement("body1"))) + body = TestBody(Seq(TestLeafStatement("body1"))) )) ) ) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingInterpreterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingInterpreterSuite.scala index 592516de84c17..384bdfe4756f6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingInterpreterSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingInterpreterSuite.scala @@ -18,9 +18,10 @@ package org.apache.spark.sql.scripting import org.apache.spark.SparkException -import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, QueryTest, Row} -import org.apache.spark.sql.catalyst.QueryPlanningTracker +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.exceptions.SqlScriptingException +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession /** @@ -29,32 +30,47 @@ import org.apache.spark.sql.test.SharedSparkSession * Output from the interpreter (iterator over executable statements) is then checked - statements * are executed and output DataFrames are compared with expected outputs. */ -class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { +class SqlScriptingInterpreterSuite extends SparkFunSuite with SharedSparkSession { // Helpers - private def runSqlScript(sqlText: String): Array[DataFrame] = { - val interpreter = SqlScriptingInterpreter() + private def verifySqlScriptResult(sqlText: String, expected: Seq[Array[Row]]): Unit = { + val interpreter = SqlScriptingInterpreter(spark) val compoundBody = spark.sessionState.sqlParser.parseScript(sqlText) - val executionPlan = interpreter.buildExecutionPlan(compoundBody, spark) - executionPlan.flatMap { - case statement: SingleStatementExec => - if (statement.isExecuted) { - None - } else { - Some(Dataset.ofRows(spark, statement.parsedPlan, new QueryPlanningTracker)) - } - case _ => None - }.toArray - } - - private def verifySqlScriptResult(sqlText: String, expected: Seq[Seq[Row]]): Unit = { - val result = runSqlScript(sqlText) + val result = interpreter.execute(compoundBody).toSeq assert(result.length == expected.length) - result.zip(expected).foreach { case (df, expectedAnswer) => checkAnswer(df, expectedAnswer) } + result.zip(expected).foreach { + case (actualAnswer, expectedAnswer) => + assert(actualAnswer.sameElements(expectedAnswer)) + } + } + + protected override def beforeAll(): Unit = { + super.beforeAll() + spark.conf.set(SQLConf.SQL_SCRIPTING_ENABLED.key, "true") + } + + protected override def afterAll(): Unit = { + spark.conf.set(SQLConf.SQL_SCRIPTING_ENABLED.key, "false") + super.afterAll() } // Tests test("select 1") { - verifySqlScriptResult("SELECT 1;", Seq(Seq(Row(1)))) + verifySqlScriptResult("SELECT 1;", Seq(Array(Row(1)))) + } + + test("select 1; select 2;") { + val sqlScript = + """ + |BEGIN + |SELECT 1; + |SELECT 2; + |END + |""".stripMargin + val expected = Seq( + Array(Row(1)), + Array(Row(2)) + ) + verifySqlScriptResult(sqlScript, expected) } test("multi statement - simple") { @@ -69,10 +85,10 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin val expected = Seq( - Seq.empty[Row], // create table - Seq.empty[Row], // insert - Seq.empty[Row], // select with filter - Seq(Row(1)) // select + Array.empty[Row], // create table + Array.empty[Row], // insert + Array.empty[Row], // select with filter + Array(Row(1)) // select ) verifySqlScriptResult(sqlScript, expected) } @@ -94,10 +110,10 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin val expected = Seq( - Seq.empty[Row], // create table - Seq.empty[Row], // insert #1 - Seq.empty[Row], // insert #2 - Seq(Row(false)) // select + Array.empty[Row], // create table + Array.empty[Row], // insert #1 + Array.empty[Row], // insert #2 + Array(Row(false)) // select ) verifySqlScriptResult(sqlScript, expected) } @@ -113,10 +129,9 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin val expected = Seq( - Seq.empty[Row], // declare var - Seq.empty[Row], // set var - Seq(Row(2)), // select - Seq.empty[Row] // drop var + Array.empty[Row], // declare var + Array.empty[Row], // set var + Array(Row(2)) // select ) verifySqlScriptResult(sqlScript, expected) } @@ -131,10 +146,9 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin val expected = Seq( - Seq.empty[Row], // declare var - Seq.empty[Row], // set var - Seq(Row(2)), // select - Seq.empty[Row] // drop var + Array.empty[Row], // declare var + Array.empty[Row], // set var + Array(Row(2)) // select ) verifySqlScriptResult(sqlScript, expected) } @@ -159,16 +173,13 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin val expected = Seq( - Seq.empty[Row], // declare var - Seq(Row(1)), // select - Seq.empty[Row], // drop var - Seq.empty[Row], // declare var - Seq(Row(2)), // select - Seq.empty[Row], // drop var - Seq.empty[Row], // declare var - Seq.empty[Row], // set var - Seq(Row(4)), // select - Seq.empty[Row] // drop var + Array.empty[Row], // declare var + Array(Row(1)), // select + Array.empty[Row], // declare var + Array(Row(2)), // select + Array.empty[Row], // declare var + Array.empty[Row], // set var + Array(Row(4)) // select ) verifySqlScriptResult(sqlScript, expected) } @@ -211,11 +222,271 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin val expected = Seq( - Seq.empty[Row], // declare var - Seq.empty[Row], // set var - Seq(Row(2)), // select - Seq.empty[Row], // drop var - explicit - Seq.empty[Row] // drop var - implicit + Array.empty[Row], // declare var + Array.empty[Row], // set var + Array(Row(2)), // select + Array.empty[Row] // drop var - explicit + ) + verifySqlScriptResult(sqlScript, expected) + } + + test("duplicate handler") { + val sqlScript = + """ + |BEGIN + | DECLARE flag INT = -1; + | DECLARE zero_division CONDITION FOR '22012'; + | DECLARE CONTINUE HANDLER FOR zero_division + | BEGIN + | SET VAR flag = 1; + | END; + | DECLARE CONTINUE HANDLER FOR '22012' + | BEGIN + | SET VAR flag = 2; + | END; + | SELECT 1/0; + | SELECT flag; + |END + |""".stripMargin + checkError( + exception = intercept[SqlScriptingException] { + verifySqlScriptResult(sqlScript, Seq.empty) + }, + errorClass = "DUPLICATE_HANDLER_FOR_SAME_SQL_STATE", + parameters = Map("sqlState" -> "22012")) + } + + test("handler - continue resolve in the same block") { + val sqlScript = + """ + |BEGIN + | DECLARE flag INT = -1; + | DECLARE zero_division CONDITION FOR '22012'; + | DECLARE CONTINUE HANDLER FOR zero_division + | BEGIN + | SELECT flag; + | SET VAR flag = 1; + | END; + | SELECT 2; + | SELECT 3; + | SELECT 1/0; + | SELECT 4; + | SELECT flag; + |END + |""".stripMargin + val expected = Seq( + Array.empty[Row], // declare var + Array(Row(2)), // select + Array(Row(3)), // select + Array(Row(-1)), // select flag + Array.empty[Row], // set flag + Array(Row(4)), // select + Array(Row(1)) // select + ) + verifySqlScriptResult(sqlScript, expected) + } + + test("handler - continue resolve in outer block") { + val sqlScript = + """ + |BEGIN + | DECLARE flag INT = -1; + | DECLARE zero_division CONDITION FOR '22012'; + | DECLARE CONTINUE HANDLER FOR zero_division + | BEGIN + | SELECT flag; + | SET VAR flag = 1; + | END; + | SELECT 2; + | BEGIN + | SELECT 3; + | BEGIN + | SELECT 4; + | SELECT 1/0; + | SELECT 5; + | END; + | SELECT 6; + | END; + | SELECT 7; + | SELECT flag; + |END + |""".stripMargin + val expected = Seq( + Array.empty[Row], // declare var + Array(Row(2)), // select + Array(Row(3)), // select + Array(Row(4)), // select + Array(Row(-1)), // select flag + Array.empty[Row], // set flag + Array(Row(5)), // select + Array(Row(6)), // select + Array(Row(7)), // select + Array(Row(1)) // select + ) + verifySqlScriptResult(sqlScript, expected) + } + + test("handler - continue resolve in the same block nested") { + val sqlScript = + """ + |BEGIN + | DECLARE flag INT = -1; + | SELECT 2; + | BEGIN + | SELECT 3; + | BEGIN + | DECLARE zero_division CONDITION FOR '22012'; + | DECLARE CONTINUE HANDLER FOR zero_division + | BEGIN + | SELECT flag; + | SET VAR flag = 1; + | END; + | SELECT 4; + | SELECT 1/0; + | SELECT 5; + | END; + | SELECT 6; + | END; + | SELECT 7; + | SELECT flag; + |END + |""".stripMargin + val expected = Seq( + Array.empty[Row], // declare var + Array(Row(2)), // select + Array(Row(3)), // select + Array(Row(4)), // select + Array(Row(-1)), // select flag + Array.empty[Row], // set flag + Array(Row(5)), // select + Array(Row(6)), // select + Array(Row(7)), // select + Array(Row(1)) // select + ) + verifySqlScriptResult(sqlScript, expected) + } + + test("handler - exit resolve in the same block") { + val sqlScript = + """ + |BEGIN + | DECLARE flag INT = -1; + | BEGIN + | DECLARE zero_division CONDITION FOR '22012'; + | DECLARE EXIT HANDLER FOR zero_division + | BEGIN + | SELECT flag; + | SET VAR flag = 1; + | END; + | SELECT 2; + | SELECT 3; + | SELECT 1/0; + | SELECT 4; + | END; + | SELECT flag; + |END + |""".stripMargin + val expected = Seq( + Array.empty[Row], // declare var + Array(Row(2)), // select + Array(Row(3)), // select + Array(Row(-1)), // select flag + Array.empty[Row], // set flag + Array(Row(1)) // select flag from the outer body + ) + verifySqlScriptResult(sqlScript, expected) + } + + test("handler - exit resolve in outer block") { + val sqlScript = + """ + |BEGIN + | DECLARE flag INT = -1; + | BEGIN + | DECLARE zero_division CONDITION FOR '22012'; + | DECLARE EXIT HANDLER FOR zero_division + | BEGIN + | SELECT flag; + | SET VAR flag = 1; + | END; + | SELECT 2; + | SELECT 3; + | BEGIN + | SELECT 4; + | SELECT 1/0; + | SELECT 5; + | END; + | SELECT 6; + | END; + | SELECT flag; + |END + |""".stripMargin + val expected = Seq( + Array.empty[Row], // declare var + Array(Row(2)), // select + Array(Row(3)), // select + Array(Row(4)), // select + Array(Row(-1)), // select flag + Array.empty[Row], // set flag + // skip select 5 + // skip select 6 + Array(Row(1)) // select flag from the outer body + ) + verifySqlScriptResult(sqlScript, expected) + } + + test("handler - continue resolve by the CATCH ALL handler") { + val sqlScript = + """ + |BEGIN + | DECLARE flag INT = -1; + | DECLARE CONTINUE HANDLER FOR SQLEXCEPTION + | BEGIN + | SELECT flag; + | SET VAR flag = 1; + | END; + | SELECT 2; + | SELECT 1/0; + | SELECT 3; + | SELECT flag; + |END + |""".stripMargin + val expected = Seq( + Array.empty[Row], // declare var + Array(Row(2)), // select + Array(Row(-1)), // select flag + Array.empty[Row], // set flag + Array(Row(3)), // select + Array(Row(1)) // select + ) + verifySqlScriptResult(sqlScript, expected) + } + + test("chained begin end blocks") { + val sqlScript = + """ + |BEGIN + | BEGIN + | SELECT 1; + | SELECT 2; + | END; + | BEGIN + | SELECT 3; + | SELECT 4; + | END; + | BEGIN + | SELECT 5; + | SELECT 6; + | END; + |END + |""".stripMargin + val expected = Seq( + Array(Row(1)), // select + Array(Row(2)), // select + Array(Row(3)), // select + Array(Row(4)), // select + Array(Row(5)), // select + Array(Row(6)) // select ) verifySqlScriptResult(sqlScript, expected) } @@ -229,7 +500,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { | END IF; |END |""".stripMargin - val expected = Seq(Seq(Row(42))) + val expected = Seq(Array(Row(42))) verifySqlScriptResult(commands, expected) } @@ -246,7 +517,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { | END IF; |END |""".stripMargin - val expected = Seq(Seq(Row(42))) + val expected = Seq(Array(Row(42))) verifySqlScriptResult(commands, expected) } @@ -263,7 +534,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin - val expected = Seq(Seq(Row(42))) + val expected = Seq(Array(Row(42))) verifySqlScriptResult(commands, expected) } @@ -283,7 +554,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin - val expected = Seq(Seq(Row(43))) + val expected = Seq(Array(Row(43))) verifySqlScriptResult(commands, expected) } @@ -300,7 +571,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin - val expected = Seq(Seq(Row(43))) + val expected = Seq(Array(Row(43))) verifySqlScriptResult(commands, expected) } @@ -320,7 +591,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin - val expected = Seq(Seq(Row(44))) + val expected = Seq(Array(Row(44))) verifySqlScriptResult(commands, expected) } @@ -340,7 +611,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin - val expected = Seq(Seq.empty[Row], Seq.empty[Row], Seq.empty[Row], Seq(Row(43))) + val expected = Seq(Array.empty[Row], Array.empty[Row], Array.empty[Row], Array(Row(43))) verifySqlScriptResult(commands, expected) } } @@ -363,7 +634,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |END |""".stripMargin - val expected = Seq(Seq.empty[Row], Seq.empty[Row], Seq.empty[Row], Seq(Row(43))) + val expected = Seq(Array.empty[Row], Array.empty[Row], Array.empty[Row], Array(Row(43))) verifySqlScriptResult(commands, expected) } } @@ -380,7 +651,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |""".stripMargin checkError( exception = intercept[SqlScriptingException] ( - runSqlScript(commands) + verifySqlScriptResult(commands, Seq()) ), errorClass = "INVALID_BOOLEAN_STATEMENT", parameters = Map("invalidStatement" -> "1") @@ -402,7 +673,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |""".stripMargin checkError( exception = intercept[SqlScriptingException] ( - runSqlScript(commands1) + verifySqlScriptResult(commands1, Seq()) ), errorClass = "BOOLEAN_STATEMENT_WITH_EMPTY_ROW", parameters = Map("invalidStatement" -> "(SELECT * FROM T1)") @@ -422,7 +693,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |""".stripMargin checkError( exception = intercept[SparkException] ( - runSqlScript(commands2) + verifySqlScriptResult(commands2, Seq()) ), errorClass = "SCALAR_SUBQUERY_TOO_MANY_ROWS", parameters = Map.empty, @@ -444,14 +715,13 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |""".stripMargin val expected = Seq( - Seq.empty[Row], // declare i - Seq(Row(0)), // select i - Seq.empty[Row], // set i - Seq(Row(1)), // select i - Seq.empty[Row], // set i - Seq(Row(2)), // select i - Seq.empty[Row], // set i - Seq.empty[Row] // drop var + Array.empty[Row], // declare i + Array(Row(0)), // select i + Array.empty[Row], // set i + Array(Row(1)), // select i + Array.empty[Row], // set i + Array(Row(2)), // select i + Array.empty[Row] // set i ) verifySqlScriptResult(commands, expected) } @@ -469,8 +739,7 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |""".stripMargin val expected = Seq( - Seq.empty[Row], // declare i - Seq.empty[Row] // drop i + Array.empty[Row] // declare i ) verifySqlScriptResult(commands, expected) } @@ -493,22 +762,20 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |""".stripMargin val expected = Seq( - Seq.empty[Row], // declare i - Seq.empty[Row], // declare j - Seq.empty[Row], // set j to 0 - Seq(Row(0, 0)), // select i, j - Seq.empty[Row], // increase j - Seq(Row(0, 1)), // select i, j - Seq.empty[Row], // increase j - Seq.empty[Row], // increase i - Seq.empty[Row], // set j to 0 - Seq(Row(1, 0)), // select i, j - Seq.empty[Row], // increase j - Seq(Row(1, 1)), // select i, j - Seq.empty[Row], // increase j - Seq.empty[Row], // increase i - Seq.empty[Row], // drop j - Seq.empty[Row] // drop i + Array.empty[Row], // declare i + Array.empty[Row], // declare j + Array.empty[Row], // set j to 0 + Array(Row(0, 0)), // select i, j + Array.empty[Row], // increase j + Array(Row(0, 1)), // select i, j + Array.empty[Row], // increase j + Array.empty[Row], // increase i + Array.empty[Row], // set j to 0 + Array(Row(1, 0)), // select i, j + Array.empty[Row], // increase j + Array(Row(1, 1)), // select i, j + Array.empty[Row], // increase j + Array.empty[Row] // increase i ) verifySqlScriptResult(commands, expected) } @@ -527,11 +794,11 @@ class SqlScriptingInterpreterSuite extends QueryTest with SharedSparkSession { |""".stripMargin val expected = Seq( - Seq.empty[Row], // create table - Seq(Row(42)), // select - Seq.empty[Row], // insert - Seq(Row(42)), // select - Seq.empty[Row] // insert + Array.empty[Row], // create table + Array(Row(42)), // select + Array.empty[Row], // insert + Array(Row(42)), // select + Array.empty[Row] // insert ) verifySqlScriptResult(commands, expected) } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala index 7005f0e951b2b..9327cbf916c33 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala @@ -214,7 +214,7 @@ trait ThriftServerWithSparkContextSuite extends SharedThriftServer { val sessionHandle = client.openSession(user, "") val infoValue = client.getInfo(sessionHandle, GetInfoType.CLI_ODBC_KEYWORDS) // scalastyle:off line.size.limit - assert(infoValue.getStringValue == "ADD,AFTER,ALL,ALTER,ALWAYS,ANALYZE,AND,ANTI,ANY,ANY_VALUE,ARCHIVE,ARRAY,AS,ASC,AT,AUTHORIZATION,BEGIN,BETWEEN,BIGINT,BINARY,BINDING,BOOLEAN,BOTH,BUCKET,BUCKETS,BY,BYTE,CACHE,CALLED,CASCADE,CASE,CAST,CATALOG,CATALOGS,CHANGE,CHAR,CHARACTER,CHECK,CLEAR,CLUSTER,CLUSTERED,CODEGEN,COLLATE,COLLATION,COLLECTION,COLUMN,COLUMNS,COMMENT,COMMIT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONSTRAINT,CONTAINS,COST,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,DATA,DATABASE,DATABASES,DATE,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAY,DAYOFYEAR,DAYS,DBPROPERTIES,DEC,DECIMAL,DECLARE,DEFAULT,DEFINED,DEFINER,DELETE,DELIMITED,DESC,DESCRIBE,DETERMINISTIC,DFS,DIRECTORIES,DIRECTORY,DISTINCT,DISTRIBUTE,DIV,DO,DOUBLE,DROP,ELSE,END,ESCAPE,ESCAPED,EVOLUTION,EXCEPT,EXCHANGE,EXCLUDE,EXECUTE,EXISTS,EXPLAIN,EXPORT,EXTENDED,EXTERNAL,EXTRACT,FALSE,FETCH,FIELDS,FILEFORMAT,FILTER,FIRST,FLOAT,FOLLOWING,FOR,FOREIGN,FORMAT,FORMATTED,FROM,FULL,FUNCTION,FUNCTIONS,GENERATED,GLOBAL,GRANT,GROUP,GROUPING,HAVING,HOUR,HOURS,IDENTIFIER,IF,IGNORE,ILIKE,IMMEDIATE,IMPORT,IN,INCLUDE,INDEX,INDEXES,INNER,INPATH,INPUT,INPUTFORMAT,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,INVOKER,IS,ITEMS,JOIN,KEYS,LANGUAGE,LAST,LATERAL,LAZY,LEADING,LEFT,LIKE,LIMIT,LINES,LIST,LOAD,LOCAL,LOCATION,LOCK,LOCKS,LOGICAL,LONG,MACRO,MAP,MATCHED,MERGE,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTE,MINUTES,MODIFIES,MONTH,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NATURAL,NO,NONE,NOT,NULL,NULLS,NUMERIC,OF,OFFSET,ON,ONLY,OPTION,OPTIONS,OR,ORDER,OUT,OUTER,OUTPUTFORMAT,OVER,OVERLAPS,OVERLAY,OVERWRITE,PARTITION,PARTITIONED,PARTITIONS,PERCENT,PIVOT,PLACING,POSITION,PRECEDING,PRIMARY,PRINCIPALS,PROPERTIES,PURGE,QUARTER,QUERY,RANGE,READS,REAL,RECORDREADER,RECORDWRITER,RECOVER,REDUCE,REFERENCES,REFRESH,RENAME,REPAIR,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURN,RETURNS,REVOKE,RIGHT,ROLE,ROLES,ROLLBACK,ROLLUP,ROW,ROWS,SCHEMA,SCHEMAS,SECOND,SECONDS,SECURITY,SELECT,SEMI,SEPARATED,SERDE,SERDEPROPERTIES,SESSION_USER,SET,SETS,SHORT,SHOW,SINGLE,SKEWED,SMALLINT,SOME,SORT,SORTED,SOURCE,SPECIFIC,SQL,START,STATISTICS,STORED,STRATIFY,STRING,STRUCT,SUBSTR,SUBSTRING,SYNC,SYSTEM_TIME,SYSTEM_VERSION,TABLE,TABLES,TABLESAMPLE,TARGET,TBLPROPERTIES,TERMINATED,THEN,TIME,TIMEDIFF,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TO,TOUCH,TRAILING,TRANSACTION,TRANSACTIONS,TRANSFORM,TRIM,TRUE,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNION,UNIQUE,UNKNOWN,UNLOCK,UNPIVOT,UNSET,UPDATE,USE,USER,USING,VALUES,VAR,VARCHAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WEEK,WEEKS,WHEN,WHERE,WHILE,WINDOW,WITH,WITHIN,X,YEAR,YEARS,ZONE") + assert(infoValue.getStringValue == "ADD,AFTER,ALL,ALTER,ALWAYS,ANALYZE,AND,ANTI,ANY,ANY_VALUE,ARCHIVE,ARRAY,AS,ASC,AT,AUTHORIZATION,BEGIN,BETWEEN,BIGINT,BINARY,BINDING,BOOLEAN,BOTH,BUCKET,BUCKETS,BY,BYTE,CACHE,CALLED,CASCADE,CASE,CAST,CATALOG,CATALOGS,CHANGE,CHAR,CHARACTER,CHECK,CLEAR,CLUSTER,CLUSTERED,CODEGEN,COLLATE,COLLATION,COLLECTION,COLUMN,COLUMNS,COMMENT,COMMIT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONDITION,CONSTRAINT,CONTAINS,CONTINUE,COST,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,DATA,DATABASE,DATABASES,DATE,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAY,DAYOFYEAR,DAYS,DBPROPERTIES,DEC,DECIMAL,DECLARE,DEFAULT,DEFINED,DEFINER,DELETE,DELIMITED,DESC,DESCRIBE,DETERMINISTIC,DFS,DIRECTORIES,DIRECTORY,DISTINCT,DISTRIBUTE,DIV,DO,DOUBLE,DROP,ELSE,END,ESCAPE,ESCAPED,EVOLUTION,EXCEPT,EXCHANGE,EXCLUDE,EXECUTE,EXISTS,EXIT,EXPLAIN,EXPORT,EXTENDED,EXTERNAL,EXTRACT,FALSE,FETCH,FIELDS,FILEFORMAT,FILTER,FIRST,FLOAT,FOLLOWING,FOR,FOREIGN,FORMAT,FORMATTED,FOUND,FROM,FULL,FUNCTION,FUNCTIONS,GENERATED,GLOBAL,GRANT,GROUP,GROUPING,HANDLER,HAVING,HOUR,HOURS,IDENTIFIER,IF,IGNORE,ILIKE,IMMEDIATE,IMPORT,IN,INCLUDE,INDEX,INDEXES,INNER,INPATH,INPUT,INPUTFORMAT,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,INVOKER,IS,ITEMS,JOIN,KEYS,LANGUAGE,LAST,LATERAL,LAZY,LEADING,LEFT,LIKE,LIMIT,LINES,LIST,LOAD,LOCAL,LOCATION,LOCK,LOCKS,LOGICAL,LONG,MACRO,MAP,MATCHED,MERGE,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTE,MINUTES,MODIFIES,MONTH,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NATURAL,NO,NONE,NOT,NULL,NULLS,NUMERIC,OF,OFFSET,ON,ONLY,OPTION,OPTIONS,OR,ORDER,OUT,OUTER,OUTPUTFORMAT,OVER,OVERLAPS,OVERLAY,OVERWRITE,PARTITION,PARTITIONED,PARTITIONS,PERCENT,PIVOT,PLACING,POSITION,PRECEDING,PRIMARY,PRINCIPALS,PROPERTIES,PURGE,QUARTER,QUERY,RANGE,READS,REAL,RECORDREADER,RECORDWRITER,RECOVER,REDUCE,REFERENCES,REFRESH,RENAME,REPAIR,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURN,RETURNS,REVOKE,RIGHT,ROLE,ROLES,ROLLBACK,ROLLUP,ROW,ROWS,SCHEMA,SCHEMAS,SECOND,SECONDS,SECURITY,SELECT,SEMI,SEPARATED,SERDE,SERDEPROPERTIES,SESSION_USER,SET,SETS,SHORT,SHOW,SINGLE,SKEWED,SMALLINT,SOME,SORT,SORTED,SOURCE,SPECIFIC,SQL,SQLEXCEPTION,START,STATISTICS,STORED,STRATIFY,STRING,STRUCT,SUBSTR,SUBSTRING,SYNC,SYSTEM_TIME,SYSTEM_VERSION,TABLE,TABLES,TABLESAMPLE,TARGET,TBLPROPERTIES,TERMINATED,THEN,TIME,TIMEDIFF,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TO,TOUCH,TRAILING,TRANSACTION,TRANSACTIONS,TRANSFORM,TRIM,TRUE,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNION,UNIQUE,UNKNOWN,UNLOCK,UNPIVOT,UNSET,UPDATE,USE,USER,USING,VALUES,VAR,VARCHAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WEEK,WEEKS,WHEN,WHERE,WHILE,WINDOW,WITH,WITHIN,X,YEAR,YEARS,ZONE") // scalastyle:on line.size.limit } }