Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-43217] Correctly recurse in nested maps/arrays in findNestedField #40879

Closed
wants to merge 2 commits into from
Closed

[SPARK-43217] Correctly recurse in nested maps/arrays in findNestedField #40879

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9dd2081
[SPARK-43217] Correctly recurse in nested maps/arrays in findNestedField
johanl-db Apr 20, 2023
d3bb9b5
nit: rename methods
johanl-db Apr 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 29 additions & 40 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,8 +328,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
resolver: Resolver = _ == _,
context: Origin = Origin()): Option[(Seq[String], StructField)] = {

@scala.annotation.tailrec
def findField(
def findFieldInStruct(
struct: StructType,
searchPath: Seq[String],
normalizedPath: Seq[String]): Option[(Seq[String], StructField)] = {
Expand All @@ -341,55 +340,45 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
} else if (found.isEmpty) {
None
} else {
val field = found.head
val currentPath = normalizedPath :+ field.name
val newSearchPath = searchPath.tail
if (newSearchPath.isEmpty) {
Some(normalizedPath -> field)
} else {
(newSearchPath, field.dataType) match {
case (_, s: StructType) =>
findField(s, newSearchPath, currentPath)

case _ if !includeCollections =>
throw QueryCompilationErrors.invalidFieldName(fieldNames, currentPath, context)

case (Seq("key", rest @ _*), MapType(keyType, _, _)) =>
findFieldInCollection(keyType, false, rest, currentPath, "key")

case (Seq("value", rest @ _*), MapType(_, valueType, isNullable)) =>
findFieldInCollection(valueType, isNullable, rest, currentPath, "value")

case (Seq("element", rest @ _*), ArrayType(elementType, isNullable)) =>
findFieldInCollection(elementType, isNullable, rest, currentPath, "element")

case _ =>
throw QueryCompilationErrors.invalidFieldName(fieldNames, currentPath, context)
}
}
findField(
parent = found.head,
searchPath = searchPath.tail,
normalizedPath)
}
}

def findFieldInCollection(
dt: DataType,
nullable: Boolean,
@scala.annotation.tailrec
def findField(
parent: StructField,
searchPath: Seq[String],
normalizedPath: Seq[String],
collectionFieldName: String): Option[(Seq[String], StructField)] = {
normalizedPath: Seq[String]): Option[(Seq[String], StructField)] = {
if (searchPath.isEmpty) {
Some(normalizedPath -> StructField(collectionFieldName, dt, nullable))
Some(normalizedPath -> parent)
} else {
val newPath = normalizedPath :+ collectionFieldName
dt match {
case s: StructType =>
findField(s, searchPath, newPath)
val currentPath = normalizedPath :+ parent.name
(searchPath, parent.dataType) match {
case (_, s: StructType) =>
findFieldInStruct(s, searchPath, currentPath)

case _ if !includeCollections =>
throw QueryCompilationErrors.invalidFieldName(fieldNames, currentPath, context)

case (Seq("key", rest @ _*), MapType(keyType, _, _)) =>
findField(StructField("key", keyType, nullable = false), rest, currentPath)

case (Seq("value", rest @ _*), MapType(_, valueType, isNullable)) =>
findField(StructField("value", valueType, isNullable), rest, currentPath)

case (Seq("element", rest @ _*), ArrayType(elementType, isNullable)) =>
findField(StructField("element", elementType, isNullable), rest, currentPath)

case _ =>
throw QueryCompilationErrors.invalidFieldName(fieldNames, newPath, context)
throw QueryCompilationErrors.invalidFieldName(fieldNames, currentPath, context)
}
}
}

findField(this, fieldNames, Nil)
findFieldInStruct(this, fieldNames, Nil)
}

protected[sql] def toAttributes: Seq[AttributeReference] = map(field => field.toAttribute)
Expand Down
105 changes: 88 additions & 17 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,8 +293,13 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
new StructType().add("a", "int"),
new StructType().add("b", "int")
))
.add("m3", MapType(IntegerType, MapType(
IntegerType,
new StructType().add("ma", IntegerType))
))
.add("a1", ArrayType(IntegerType))
.add("a2", ArrayType(new StructType().add("c", "int")))
.add("a3", ArrayType(ArrayType(new StructType().add("d", "int"))))

def check(field: Seq[String], expect: Option[(Seq[String], StructField)]): Unit = {
val res = input.findNestedField(field, resolver = caseInsensitiveResolution)
Expand Down Expand Up @@ -322,8 +327,12 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
var e = intercept[AnalysisException] {
check(Seq("S1", "S12", "S123"), None)
}
assert(e.getMessage.contains(
"Field name `S1`.`S12`.`S123` is invalid: `s1`.`s12` is not a struct"))
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`S1`.`S12`.`S123`",
"path" -> "`s1`.`s12`"))

// ambiguous name
e = intercept[AnalysisException] {
Expand All @@ -339,19 +348,32 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
e = intercept[AnalysisException] {
check(Seq("m1", "key"), None)
}
assert(e.getMessage.contains("Field name `m1`.`key` is invalid: `m1` is not a struct"))
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`m1`.`key`",
"path" -> "`m1`"))
checkCollection(Seq("m1", "key"), Some(Seq("m1") -> StructField("key", IntegerType, false)))
checkCollection(Seq("M1", "value"), Some(Seq("m1") -> StructField("value", IntegerType)))
e = intercept[AnalysisException] {
checkCollection(Seq("M1", "key", "name"), None)
}
assert(e.getMessage.contains(
"Field name `M1`.`key`.`name` is invalid: `m1`.`key` is not a struct"))
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`M1`.`key`.`name`",
"path" -> "`m1`.`key`"))
e = intercept[AnalysisException] {
checkCollection(Seq("M1", "value", "name"), None)
}
assert(e.getMessage.contains(
"Field name `M1`.`value`.`name` is invalid: `m1`.`value` is not a struct"))
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`M1`.`value`.`name`",
"path" -> "`m1`.`value`"))

// map of struct
checkCollection(Seq("M2", "key", "A"),
Expand All @@ -363,25 +385,41 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
e = intercept[AnalysisException] {
checkCollection(Seq("m2", "key", "A", "name"), None)
}
assert(e.getMessage.contains(
"Field name `m2`.`key`.`A`.`name` is invalid: `m2`.`key`.`a` is not a struct"))
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`m2`.`key`.`A`.`name`",
"path" -> "`m2`.`key`.`a`"))
e = intercept[AnalysisException] {
checkCollection(Seq("M2", "value", "b", "name"), None)
}
assert(e.getMessage.contains(
"Field name `M2`.`value`.`b`.`name` is invalid: `m2`.`value`.`b` is not a struct"))

checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`M2`.`value`.`b`.`name`",
"path" -> "`m2`.`value`.`b`"))
// simple array type
e = intercept[AnalysisException] {
check(Seq("A1", "element"), None)
}
assert(e.getMessage.contains("Field name `A1`.`element` is invalid: `a1` is not a struct"))
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`A1`.`element`",
"path" -> "`a1`"))
checkCollection(Seq("A1", "element"), Some(Seq("a1") -> StructField("element", IntegerType)))
e = intercept[AnalysisException] {
checkCollection(Seq("A1", "element", "name"), None)
}
assert(e.getMessage.contains(
"Field name `A1`.`element`.`name` is invalid: `a1`.`element` is not a struct"))
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`A1`.`element`.`name`",
"path" -> "`a1`.`element`"))

// array of struct
checkCollection(Seq("A2", "element", "C"),
Expand All @@ -390,8 +428,41 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
e = intercept[AnalysisException] {
checkCollection(Seq("a2", "element", "C", "name"), None)
}
assert(e.getMessage.contains(
"Field name `a2`.`element`.`C`.`name` is invalid: `a2`.`element`.`c` is not a struct"))
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`a2`.`element`.`C`.`name`",
"path" -> "`a2`.`element`.`c`"))

// nested maps
checkCollection(Seq("M3", "value", "value", "MA"),
Some(Seq("m3", "value", "value") -> StructField("ma", IntegerType)))
checkCollection(Seq("M3", "value", "value", "non_exist"), None)
e = intercept[AnalysisException] {
checkCollection(Seq("M3", "value", "value", "MA", "name"), None)
}
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`M3`.`value`.`value`.`MA`.`name`",
"path" -> "`m3`.`value`.`value`.`ma`"))

// nested arrays
checkCollection(Seq("A3", "element", "element", "D"),
Some(Seq("a3", "element", "element") -> StructField("d", IntegerType)))
checkCollection(Seq("A3", "element", "element", "non_exist"), None)
e = intercept[AnalysisException] {
checkCollection(Seq("A3", "element", "element", "D", "name"), None)
}
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`A3`.`element`.`element`.`D`.`name`",
"path" -> "`a3`.`element`.`element`.`d`")
)
}

test("SPARK-36807: Merge ANSI interval types to a tightest common type") {
Expand Down