Skip to content

Commit

Permalink
[SPARK-47265][SQL][FOLLOWUP] V2SessionCatalog#createTable should not …
Browse files Browse the repository at this point in the history
…fail

### What changes were proposed in this pull request?

This is a followup of apache#45368 . `DelegatingCatalogExtension` can invoke any method in `V2SessionCatalog`, and we should make sure the two overloads of `createTable` in `V2SessionCatalog` both work.

### Why are the changes needed?

Avoid breaking custom catalogs that extend `DelegatingCatalogExtension`.

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

N/A

### Was this patch authored or co-authored using generative AI tooling?

no

Closes apache#47755 from cloud-fan/create-table.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Kent Yao <yao@apache.org>
  • Loading branch information
cloud-fan authored and attilapiros committed Oct 4, 2024
1 parent 7cf7e7f commit 130a6f6
Showing 1 changed file with 15 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import org.apache.spark.sql.internal.connector.V1Function
import org.apache.spark.sql.types.{DataType, StructType}
import org.apache.spark.sql.util.CaseInsensitiveStringMap
import org.apache.spark.util.ArrayImplicits._
import org.apache.spark.util.Utils

/**
* A [[TableCatalog]] that translates calls to the v1 SessionCatalog.
Expand Down Expand Up @@ -158,9 +159,9 @@ class V2SessionCatalog(catalog: SessionCatalog)
catalog.refreshTable(ident.asTableIdentifier)
}

override def createTable(
private def createTable0(
ident: Identifier,
columns: Array[Column],
schema: StructType,
partitions: Array[Transform],
properties: util.Map[String, String]): Table = {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
Expand All @@ -178,7 +179,6 @@ class V2SessionCatalog(catalog: SessionCatalog)
CatalogTableType.MANAGED
}

val schema = CatalogV2Util.v2ColumnsToStructType(columns)
val (newSchema, newPartitions) = DataSourceV2Utils.getTableProvider(provider, conf) match {
// If the provider does not support external metadata, users should not be allowed to
// specify custom schema when creating the data source table, since the schema will not
Expand Down Expand Up @@ -255,7 +255,18 @@ class V2SessionCatalog(catalog: SessionCatalog)
schema: StructType,
partitions: Array[Transform],
properties: util.Map[String, String]): Table = {
throw QueryCompilationErrors.createTableDeprecatedError()
if (Utils.isTesting) {
throw QueryCompilationErrors.createTableDeprecatedError()
}
createTable0(ident, schema, partitions, properties)
}

override def createTable(
ident: Identifier,
columns: Array[Column],
partitions: Array[Transform],
properties: util.Map[String, String]): Table = {
createTable0(ident, CatalogV2Util.v2ColumnsToStructType(columns), partitions, properties)
}

private def toOptions(properties: Map[String, String]): Map[String, String] = {
Expand Down

0 comments on commit 130a6f6

Please sign in to comment.