[Spark] Fix compile error on Spark master due to ParserInterface chan…

…ges (#3294)

## Description
apache/spark#46665 introduced a new
`parseScript` method in `ParserInterface`, which broke compilation
against Spark master:
```
[error] /home/runner/work/delta/delta/spark/src/main/scala/io/delta/sql/parser/DeltaSqlParser.scala:75:7: class DeltaSqlParser needs to be abstract.
[error] Missing implementation for member of trait ParserInterface:
[error]   def parseScript(sqlScriptText: String): org.apache.spark.sql.catalyst.parser.CompoundBody = ???
[error] class DeltaSqlParser(val delegate: ParserInterface) extends ParserInterface {
[error]       ^
[warn] 100 warnings found
[error] one error found
[error] (spark / Compile / compileIncremental) Compilation failed
```

This PR fixes the issue by shimming the `ParserInterface` since
`parseScript` is not available in Spark 3.5
## How was this patch tested?
Existing DeltaSqlParserSuite

spark/src/main/scala-spark-3.5/shims/ParserInterfaceShims.scala

@@ -0,0 +1,52 @@
+/*
+ * Copyright (2021) The Delta Lake Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.parser
+
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.types.{DataType, StructType}
+
+trait ParserInterfaceShims extends ParserInterface {
+  // Handles a breaking change between Spark 3.5 and Spark Master (4.0).
+  // In Spark 4.0, `parseScript` is added to ParserInterface.
+  def parseScript(sqlScriptText: String): CompoundBody =
+    throw new UnsupportedOperationException("This method is not defined in Spark 3.5")
+}
+
+object ParserInterfaceShims {
+  def apply(parser: ParserInterface): ParserInterfaceShims = new ParserInterfaceShims {
+    override def parsePlan(sqlText: String): LogicalPlan = parser.parsePlan(sqlText)
+
+    override def parseQuery(sqlText: String): LogicalPlan = parser.parseQuery(sqlText)
+
+    override def parseExpression(sqlText: String): Expression = parser.parseExpression(sqlText)
+
+    override def parseTableIdentifier(sqlText: String): TableIdentifier =
+      parser.parseTableIdentifier(sqlText)
+
+    override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier =
+      parser.parseFunctionIdentifier(sqlText)
+
+    override def parseMultipartIdentifier (sqlText: String): Seq[String] =
+      parser.parseMultipartIdentifier(sqlText)
+
+    override def parseTableSchema(sqlText: String): StructType = parser.parseTableSchema(sqlText)
+
+    override def parseDataType(sqlText: String): DataType = parser.parseDataType(sqlText)
+  }
+}

spark/src/main/scala-spark-3.5/shims/SqlScriptingLogicalOperatorsShims.scala

@@ -0,0 +1,31 @@
+/*
+ * Copyright (2021) The Delta Lake Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.parser
+
+// Handles a breaking change between Spark 3.5 and Spark Master (4.0).
+// `CompoundBody` is a new class in Spark 4.0.
+/**
+ * Trait for all SQL Scripting logical operators that are product of parsing phase.
+ * These operators will be used by the SQL Scripting interpreter to generate execution nodes.
+ */
+sealed trait CompoundPlanStatement
+
+/**
+ * Logical operator for a compound body. Contains all statements within the compound body.
+ * @param collection Collection of statements within the compound body.
+ */
+case class CompoundBody(collection: Seq[CompoundPlanStatement]) extends CompoundPlanStatement

spark/src/main/scala-spark-master/shims/ParserInterfaceShims.scala

@@ -0,0 +1,50 @@
+/*
+ * Copyright (2021) The Delta Lake Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.parser
+
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.types.{DataType, StructType}
+
+trait ParserInterfaceShims extends ParserInterface
+
+object ParserInterfaceShims {
+  def apply(parser: ParserInterface): ParserInterfaceShims = new ParserInterfaceShims {
+    override def parsePlan(sqlText: String): LogicalPlan = parser.parsePlan(sqlText)
+
+    override def parseQuery(sqlText: String): LogicalPlan = parser.parseQuery(sqlText)
+
+    override def parseExpression(sqlText: String): Expression = parser.parseExpression(sqlText)
+
+    override def parseTableIdentifier(sqlText: String): TableIdentifier =
+      parser.parseTableIdentifier(sqlText)
+
+    override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier =
+      parser.parseFunctionIdentifier(sqlText)
+
+    override def parseMultipartIdentifier (sqlText: String): Seq[String] =
+      parser.parseMultipartIdentifier(sqlText)
+
+    override def parseTableSchema(sqlText: String): StructType = parser.parseTableSchema(sqlText)
+
+    override def parseDataType(sqlText: String): DataType = parser.parseDataType(sqlText)
+
+    override def parseScript(sqlScriptText: String): CompoundBody =
+      parser.parseScript(sqlScriptText)
+  }
+}

spark/src/main/scala/io/delta/sql/parser/DeltaSqlParser.scala

@@ -59,7 +59,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, Literal}
 import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.analysis.UnresolvedTableImplicits._
-import org.apache.spark.sql.catalyst.parser.{ParseErrorListener, ParseException, ParserInterface}
+import org.apache.spark.sql.catalyst.parser.{CompoundBody, ParseErrorListener, ParseException, ParserInterface, ParserInterfaceShims}
 import org.apache.spark.sql.catalyst.parser.ParserUtils.{checkDuplicateClauses, string, withOrigin}
 import org.apache.spark.sql.catalyst.plans.logical.{AlterTableAddConstraint, AlterTableDropConstraint, AlterTableDropFeature, CloneTableStatement, LogicalPlan, RestoreTableStatement}
 import org.apache.spark.sql.catalyst.trees.Origin
@@ -72,7 +72,8 @@ import org.apache.spark.sql.types._
  * A SQL parser that tries to parse Delta commands. If failing to parse the SQL text, it will
  * forward the call to `delegate`.
  */
-class DeltaSqlParser(val delegate: ParserInterface) extends ParserInterface {
+class DeltaSqlParser(val delegateSpark: ParserInterface) extends ParserInterfaceShims {
+  private val delegate = ParserInterfaceShims(delegateSpark)
   private val builder = new DeltaSqlAstBuilder
   private val substitution = new VariableSubstitution
 
@@ -155,6 +156,9 @@ class DeltaSqlParser(val delegate: ParserInterface) extends ParserInterface {
   override def parseTableSchema(sqlText: String): StructType = delegate.parseTableSchema(sqlText)
 
   override def parseDataType(sqlText: String): DataType = delegate.parseDataType(sqlText)
+
+  override def parseScript(sqlScriptText: String): CompoundBody =
+    delegate.parseScript(sqlScriptText)
 }
 
 /**