Skip to content

Commit

Permalink
[SPARK-44873] Support alter view with nested columns in Hive client
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?
Previously, if a view's schema contains a nested struct, alterTable using Hive client will fail. This change supports a view with a nested struct. The mechanism is to store an empty schema when we call Hive client, since we already store the actual schema in table properties. This fix is similar to apache#37364

### Why are the changes needed?
This supports using view with nested structs in Hive metastore.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added unit test.

Closes apache#42532 from kylerong-db/hive_view.

Authored-by: kylerong-db <kyle.rong@databricks.com>
Signed-off-by: Gengliang Wang <gengliang@apache.org>
  • Loading branch information
kylerong-db authored and gengliangwang committed Aug 18, 2023
1 parent 8fb799d commit c7d63ea
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -595,7 +595,16 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat

if (tableDefinition.tableType == VIEW) {
val newTableProps = tableDefinition.properties ++ tableMetaToTableProps(tableDefinition).toMap
client.alterTable(tableDefinition.copy(properties = newTableProps))
val newTable = tableDefinition.copy(properties = newTableProps)
try {
client.alterTable(newTable)
} catch {
case NonFatal(e) =>
// If for some reason we fail to store the schema we store it as empty there
// since we already store the real schema in the table properties. This try-catch
// should only be necessary for Spark views which are incompatible with Hive
client.alterTable(newTable.copy(schema = EMPTY_DATA_SCHEMA))
}
} else {
val oldTableDef = getRawTable(db, tableDefinition.identifier.table)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -385,4 +385,14 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest with ParquetTest {
checkAnswer(spark.table("t"), Row(Row("a", 1)))
}
}

test("Alter view with nested struct") {
withView("t", "t2") {
sql("CREATE OR REPLACE VIEW t AS SELECT " +
"struct(id AS `$col2`, struct(id AS `$col`) AS s1) AS s2 FROM RANGE(5)")
sql("ALTER VIEW t SET TBLPROPERTIES ('x' = 'y')")
sql("ALTER VIEW t RENAME TO t2")
checkAnswer(sql("show TBLPROPERTIES t2 (x)"), Row("x", "y"))
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -175,14 +175,6 @@ class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeA
withView("v") {
spark.sql("CREATE VIEW v AS SELECT STRUCT('a' AS `a`, 1 AS b) q")
checkAnswer(sql("SELECT q.`a`, q.b FROM v"), Row("a", 1) :: Nil)

checkError(
exception = intercept[SparkException] {
spark.sql("ALTER VIEW v AS SELECT STRUCT('a' AS `$a`, 1 AS b) q")
},
errorClass = "CANNOT_RECOGNIZE_HIVE_TYPE",
parameters = Map("fieldType" -> "\"STRUCT<$A:STRING,B:INT>\"", "fieldName" -> "`q`")
)
}
}

Expand Down

0 comments on commit c7d63ea

Please sign in to comment.