[CARMEL-1440][FOLLOWUP] Fix the codegen issue (delta-io#2)

LantaoJin · Nov 6, 2019 · 2f49adf · 2f49adf
1 parent 5b82307
commit 2f49adf
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 5 deletions.
diff --git a/build.sbt b/build.sbt
@@ -24,10 +24,15 @@ scalaVersion := crossScalaVersions.value.head
 
 sparkVersion := "2.3.0"
 
+val parquetVersion = "1.8.3-encryption-ebay0.1-SNAPSHOT"
+val parquetFormatVersion = "2.3.1-encryption-ebay0.1-SNAPSHOT"
+
 unmanagedBase := baseDirectory.value / "lib"
 
 libraryDependencies ++= Seq(
   // Adding test classifier seems to break transitive resolution of the core dependencies
+  "org.apache.parquet" % "parquet-column" % parquetVersion % "provided",
+  "org.apache.parquet" % "parquet-hadoop" % parquetVersion % "provided",
   "org.apache.spark" %% "spark-hive" % sparkVersion.value % "provided",
   "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
   "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
@@ -243,3 +248,8 @@ releaseProcess := Seq[ReleaseStep](
   setNextVersion,
   commitNextVersion
 )
+
+resolvers ++= Seq(
+  "ebaycentral.snapshots" at "http://ebaycentral.qa.ebay.com/content/repositories/snapshots",
+  "ebaycentral.releases"  at "http://ebaycentral.qa.ebay.com/content/repositories/releases"
+)
diff --git a/src/main/scala/io/delta/sql/analysis/PreprocessTableUpdate.scala b/src/main/scala/io/delta/sql/analysis/PreprocessTableUpdate.scala
@@ -38,10 +38,11 @@ class PreprocessTableUpdateDelete(
 
       val targetColNameParts =
         update.updateColumns.map{col => new UnresolvedAttribute(col.name.split("\\.")).nameParts}
-
       val alignedUpdateExprs = generateUpdateExpressions(
         update.child.output, targetColNameParts, update.updateExpressions, conf.resolver)
-      UpdateCommand(index, update.child, alignedUpdateExprs, update.condition)
+      val command = UpdateCommand(index, update.child, alignedUpdateExprs, update.condition)
+      spark.sessionState.analyzer.checkAnalysis(command)
+      command
 
     case delete: Delete =>
       val index = EliminateSubqueryAliases(delete.child) match {
@@ -50,7 +51,9 @@ class PreprocessTableUpdateDelete(
         case o =>
           throw DeltaErrors.notADeltaSourceException("DELETE", Some(o))
       }
-      DeleteCommand(index, delete.child, delete.condition)
+      val command = DeleteCommand(index, delete.child, delete.condition)
+      spark.sessionState.analyzer.checkAnalysis(command)
+      command
   }
 
   override def conf: SQLConf = spark.sessionState.conf

diff --git a/src/main/scala/org/apache/spark/sql/delta/schema/CheckDeltaInvariant.scala b/src/main/scala/org/apache/spark/sql/delta/schema/CheckDeltaInvariant.scala
@@ -57,7 +57,7 @@ case class CheckDeltaInvariant(
   private def generateNotNullCode(ctx: CodegenContext): String = {
     val childGen = child.genCode(ctx)
     val invariantField = ctx.addReferenceObj("errMsg", invariant)
-    """${childGen.code}
+    s"""${childGen.code}
        |
        |if (${childGen.isNull}) {
        |  throw org.apache.spark.sql.delta.schema.InvariantViolationException.apply(
@@ -74,7 +74,7 @@ case class CheckDeltaInvariant(
     val childGen = resolvedExpr.genCode(ctx)
     val invariantField = ctx.addReferenceObj("errMsg", invariant)
     val eValue = ctx.freshName("elementResult")
-    """${elementValue.code}
+    s"""${elementValue.code}
        |${childGen.code}
        |
        |if (${childGen.isNull} || ${childGen.value} == false) {