Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SQL] Fixed expression data type matching. #5675

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
org.apache.spark.sql.types.UTF8String(${eval.primitiveTerm}.toString)
""".children

case EqualTo(e1: BinaryType, e2: BinaryType) =>
case EqualTo(e1 @ BinaryType(), e2 @ BinaryType()) =>
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a bug introduced in #5350

(e1, e2).evaluateAs (BooleanType) {
case (eval1, eval2) =>
q"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,32 +40,46 @@ import org.apache.spark.util.Utils
*/
@DeveloperApi
abstract class DataType {
/** Matches any expression that evaluates to this DataType */
def unapply(a: Expression): Boolean = a match {
/**
* Enables matching against NumericType for expressions:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo? Seems it should be DataType.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah yes - I will fix that.

* {{{
* case Cast(child @ BinaryType(), StringType) =>
* ...
* }}}
*/
private[sql] def unapply(a: Expression): Boolean = a match {
case e: Expression if e.dataType == this => true
case _ => false
}

/** The default size of a value of this data type. */
/**
* The default size of a value of this data type, used internally for size estimation.
*/
def defaultSize: Int

/** Name of the type used in JSON serialization. */
def typeName: String = this.getClass.getSimpleName.stripSuffix("$").dropRight(4).toLowerCase

private[sql] def jsonValue: JValue = typeName

/** The compact JSON representation of this data type. */
def json: String = compact(render(jsonValue))

/** The pretty (i.e. indented) JSON representation of this data type. */
def prettyJson: String = pretty(render(jsonValue))

/** Readable string representation for the type. */
def simpleString: String = typeName

/** Check if `this` and `other` are the same data type when ignoring nullability
* (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
/**
* Check if `this` and `other` are the same data type when ignoring nullability
* (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
*/
private[spark] def sameType(other: DataType): Boolean =
DataType.equalsIgnoreNullability(this, other)

/** Returns the same data type but set all nullability fields are true
/**
* Returns the same data type but set all nullability fields are true
* (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
*/
private[spark] def asNullable: DataType
Expand Down Expand Up @@ -104,12 +118,25 @@ abstract class NumericType extends AtomicType {


private[sql] object NumericType {
/**
* Enables matching against NumericType for expressions:
* {{{
* case Cast(child @ NumericType(), StringType) =>
* ...
* }}}
*/
def unapply(e: Expression): Boolean = e.dataType.isInstanceOf[NumericType]
}


/** Matcher for any expressions that evaluate to [[IntegralType]]s */
private[sql] object IntegralType {
/**
* Enables matching against IntegralType for expressions:
* {{{
* case Cast(child @ IntegralType(), StringType) =>
* ...
* }}}
*/
def unapply(a: Expression): Boolean = a match {
case e: Expression if e.dataType.isInstanceOf[IntegralType] => true
case _ => false
Expand All @@ -122,9 +149,14 @@ private[sql] abstract class IntegralType extends NumericType {
}



/** Matcher for any expressions that evaluate to [[FractionalType]]s */
private[sql] object FractionalType {
/**
* Enables matching against FractionalType for expressions:
* {{{
* case Cast(child @ FractionalType(), StringType) =>
* ...
* }}}
*/
def unapply(a: Expression): Boolean = a match {
case e: Expression if e.dataType.isInstanceOf[FractionalType] => true
case _ => false
Expand Down