Skip to content

Commit

Permalink
[SPARK-4825] [SQL] CTAS fails to resolve when created using saveAsTable
Browse files Browse the repository at this point in the history
Fix bug when query like:
```
  test("save join to table") {
    val testData = sparkContext.parallelize(1 to 10).map(i => TestData(i, i.toString))
    sql("CREATE TABLE test1 (key INT, value STRING)")
    testData.insertInto("test1")
    sql("CREATE TABLE test2 (key INT, value STRING)")
    testData.insertInto("test2")
    testData.insertInto("test2")
    sql("SELECT COUNT(a.value) FROM test1 a JOIN test2 b ON a.key = b.key").saveAsTable("test")
    checkAnswer(
      table("test"),
      sql("SELECT COUNT(a.value) FROM test1 a JOIN test2 b ON a.key = b.key").collect().toSeq)
  }
```

Author: Cheng Hao <hao.cheng@intel.com>

Closes #3673 from chenghao-intel/spark_4825 and squashes the following commits:

e8cbd56 [Cheng Hao] alternate the pattern matching order for logical plan:CTAS
e004895 [Cheng Hao] fix bug

(cherry picked from commit 0abbff2)
Signed-off-by: Michael Armbrust <michael@databricks.com>
  • Loading branch information
chenghao-intel authored and marmbrus committed Dec 12, 2014
1 parent c3b0713 commit c82e99d
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ case class CreateTableAsSelect[T](
allowExisting: Boolean,
desc: Option[T] = None) extends UnaryNode {
override def output = Seq.empty[Attribute]
override lazy val resolved = (databaseName != None && childrenResolved)
override lazy val resolved = databaseName != None && childrenResolved
}

case class WriteToFile(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,8 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with
// Wait until children are resolved.
case p: LogicalPlan if !p.childrenResolved => p

// TODO extra is in type of ASTNode which means the logical plan is not resolved
// Need to think about how to implement the CreateTableAsSelect.resolved
case CreateTableAsSelect(db, tableName, child, allowExisting, Some(extra: ASTNode)) =>
val (dbName, tblName) = processDatabaseAndTableName(db, tableName)
val databaseName = dbName.getOrElse(hive.sessionState.getCurrentDatabase)
Expand All @@ -285,6 +287,13 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with
}

CreateTableAsSelect(Some(databaseName), tblName, child, allowExisting, desc)

case p: LogicalPlan if p.resolved => p

case p @ CreateTableAsSelect(db, tableName, child, allowExisting, None) =>
val (dbName, tblName) = processDatabaseAndTableName(db, tableName)
val databaseName = dbName.getOrElse(hive.sessionState.getCurrentDatabase)
CreateTableAsSelect(Some(databaseName), tblName, child, allowExisting, None)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,19 @@ class SQLQuerySuite extends QueryTest {
sql("SELECT key, value FROM src ORDER BY key").collect().toSeq)
}

test("SPARK-4825 save join to table") {
val testData = sparkContext.parallelize(1 to 10).map(i => TestData(i, i.toString))
sql("CREATE TABLE test1 (key INT, value STRING)")
testData.insertInto("test1")
sql("CREATE TABLE test2 (key INT, value STRING)")
testData.insertInto("test2")
testData.insertInto("test2")
sql("SELECT COUNT(a.value) FROM test1 a JOIN test2 b ON a.key = b.key").saveAsTable("test")
checkAnswer(
table("test"),
sql("SELECT COUNT(a.value) FROM test1 a JOIN test2 b ON a.key = b.key").collect().toSeq)
}

test("SPARK-3708 Backticks aren't handled correctly is aliases") {
checkAnswer(
sql("SELECT k FROM (SELECT `key` AS `k` FROM src) a"),
Expand Down

0 comments on commit c82e99d

Please sign in to comment.