-
Notifications
You must be signed in to change notification settings - Fork 28.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[SPARK-8203] [SPARK-8204] [SQL] conditional function: least/greatest #6851
Changes from 3 commits
ec625b0
c1f6824
7a6bdbb
0f1bff2
86fb049
22e8f3d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -20,7 +20,8 @@ package org.apache.spark.sql.catalyst.expressions | |
import org.apache.spark.sql.catalyst.InternalRow | ||
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult | ||
import org.apache.spark.sql.catalyst.expressions.codegen._ | ||
import org.apache.spark.sql.types.{BooleanType, DataType} | ||
import org.apache.spark.sql.catalyst.util.TypeUtils | ||
import org.apache.spark.sql.types.{NullType, BooleanType, DataType} | ||
|
||
|
||
case class If(predicate: Expression, trueValue: Expression, falseValue: Expression) | ||
|
@@ -312,3 +313,103 @@ case class CaseKeyWhen(key: Expression, branches: Seq[Expression]) extends CaseW | |
}.mkString | ||
} | ||
} | ||
|
||
case class Least(children: Expression*) extends Expression { | ||
require(children.length > 1, "LEAST requires at least 2 arguments, got " + children.length) | ||
|
||
override def nullable: Boolean = children.forall(_.nullable) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. shouldn't least be nullable if any is nullable? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. hive 1.2.0 only return null when all arguments are null. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If any child is not nullable, the result will not be nullable. |
||
override def foldable: Boolean = children.forall(_.foldable) | ||
|
||
private lazy val ordering = TypeUtils.getOrdering(dataType) | ||
|
||
override def checkInputDataTypes(): TypeCheckResult = { | ||
if (children.map(_.dataType).distinct.count(_ != NullType) > 1) { | ||
TypeCheckResult.TypeCheckFailure( | ||
s"The expressions should all have the same type," + | ||
s" got LEAST (${children.map(_.dataType)}).") | ||
} else { | ||
TypeUtils.checkForOrderingExpr(dataType, "function " + prettyName) | ||
} | ||
} | ||
|
||
override def dataType: DataType = children.head.dataType | ||
|
||
override def eval(input: InternalRow): Any = { | ||
children.foldLeft[Any](null)((r, c) => { | ||
val evalc = c.eval(input) | ||
if (evalc != null) { | ||
if (r == null || ordering.lt(evalc, r)) evalc else r | ||
} else { | ||
r | ||
} | ||
}) | ||
} | ||
|
||
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { | ||
val evalChildren = children.map(_.gen(ctx)) | ||
def updateEval(i: Int): String = | ||
s""" | ||
if (${ev.isNull} || (!${evalChildren(i).isNull} && ${ | ||
ctx.genComp(dataType, evalChildren(i).primitive, ev.primitive)} < 0)) { | ||
${ev.isNull} = ${evalChildren(i).isNull}; | ||
${ev.primitive} = ${evalChildren(i).primitive}; | ||
} | ||
""" | ||
s""" | ||
${evalChildren.map(_.code).mkString("\n")} | ||
boolean ${ev.isNull} = true; | ||
${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; | ||
${(0 to children.length - 1).map(updateEval).mkString("\n")} | ||
""" | ||
} | ||
} | ||
|
||
case class Greatest(children: Expression*) extends Expression { | ||
require(children.length > 1, "GREATEST requires at least 2 arguments, got " + children.length) | ||
|
||
override def nullable: Boolean = children.forall(_.nullable) | ||
override def foldable: Boolean = children.forall(_.foldable) | ||
|
||
private lazy val ordering = TypeUtils.getOrdering(dataType) | ||
|
||
override def checkInputDataTypes(): TypeCheckResult = { | ||
if (children.map(_.dataType).distinct.count(_ != NullType) > 1) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why count against not There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i think the problem is that null literals are not converted into any non-null type when it happens in greatest. |
||
TypeCheckResult.TypeCheckFailure( | ||
s"The expressions should all have the same type," + | ||
s" got GREATEST (${children.map(_.dataType)}).") | ||
} else { | ||
TypeUtils.checkForOrderingExpr(dataType, "function " + prettyName) | ||
} | ||
} | ||
|
||
override def dataType: DataType = children.head.dataType | ||
|
||
override def eval(input: InternalRow): Any = { | ||
children.foldLeft[Any](null)((r, c) => { | ||
val evalc = c.eval(input) | ||
if (evalc != null) { | ||
if (r == null || ordering.gt(evalc, r)) evalc else r | ||
} else { | ||
r | ||
} | ||
}) | ||
} | ||
|
||
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { | ||
val evalChildren = children.map(_.gen(ctx)) | ||
def updateEval(i: Int): String = | ||
s""" | ||
if (${ev.isNull} || (!${evalChildren(i).isNull} && ${ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could you move |
||
ctx.genComp(dataType, evalChildren(i).primitive, ev.primitive)} > 0)) { | ||
${ev.isNull} = ${evalChildren(i).isNull}; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if ev.isNull, this will be executed. |
||
${ev.primitive} = ${evalChildren(i).primitive}; | ||
} | ||
""" | ||
s""" | ||
${evalChildren.map(_.code).mkString("\n")} | ||
boolean ${ev.isNull} = true; | ||
${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; | ||
${(0 to children.length - 1).map(updateEval).mkString("\n")} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
""" | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -599,7 +599,7 @@ object functions { | |
/** | ||
* Creates a new row for each element in the given array or map column. | ||
*/ | ||
def explode(e: Column): Column = Explode(e.expr) | ||
def explode(e: Column): Column = Explode(e.expr) | ||
|
||
/** | ||
* Converts a string exprsesion to lower case. | ||
|
@@ -1073,15 +1073,34 @@ object functions { | |
def floor(columnName: String): Column = floor(Column(columnName)) | ||
|
||
/** | ||
* Computes hex value of the given column | ||
* Returns the greatest value of the list of values. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should also document semantics regarding null values |
||
* | ||
* @group math_funcs | ||
* @group normal_funcs | ||
* @since 1.5.0 | ||
*/ | ||
@scala.annotation.varargs | ||
def greatest(exprs: Column*): Column = Greatest(exprs.map(_.expr): _*) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we check that |
||
|
||
/** | ||
* Returns the greatest value of the list of column names. | ||
* | ||
* @group normal_funcs | ||
* @since 1.5.0 | ||
*/ | ||
@scala.annotation.varargs | ||
def greatest(columnName: String, columnNames: String*): Column = | ||
greatest((columnName +: columnNames).map(Column.apply): _*) | ||
|
||
/** | ||
* Computes hex value of the given column. | ||
* | ||
* @group math_funcs | ||
* @since 1.5.0 | ||
*/ | ||
def hex(column: Column): Column = Hex(column.expr) | ||
|
||
/** | ||
* Computes hex value of the given input | ||
* Computes hex value of the given input. | ||
* | ||
* @group math_funcs | ||
* @since 1.5.0 | ||
|
@@ -1171,6 +1190,25 @@ object functions { | |
*/ | ||
def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName)) | ||
|
||
/** | ||
* Returns the least value of the list of values. | ||
* | ||
* @group normal_funcs | ||
* @since 1.5.0 | ||
*/ | ||
@scala.annotation.varargs | ||
def least(exprs: Column*): Column = Least(exprs.map(_.expr): _*) | ||
|
||
/** | ||
* Returns the least value of the list of column names. | ||
* | ||
* @group normal_funcs | ||
* @since 1.5.0 | ||
*/ | ||
@scala.annotation.varargs | ||
def least(columnName: String, columnNames: String*): Column = | ||
least((columnName +: columnNames).map(Column.apply): _*) | ||
|
||
/** | ||
* Computes the natural logarithm of the given value. | ||
* | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
note: this should be Seq[Expression], not vararg