-
Notifications
You must be signed in to change notification settings - Fork 32
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Translate PPL-builtin functions to Spark-builtin functions #448
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.sql.ppl.utils; | ||
|
||
import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction; | ||
import org.apache.spark.sql.catalyst.expressions.Expression; | ||
import org.opensearch.sql.expression.function.BuiltinFunctionName; | ||
|
||
import java.util.List; | ||
import java.util.Locale; | ||
|
||
import static org.opensearch.sql.ppl.utils.DataTypeTransformer.seq; | ||
import static scala.Option.empty; | ||
|
||
public interface BuiltinFunctionTranslator { | ||
|
||
static Expression builtinFunction(org.opensearch.sql.ast.expression.Function function, List<Expression> args) { | ||
if (BuiltinFunctionName.of(function.getFuncName()).isEmpty()) { | ||
// TODO change it when UDF is supported | ||
// TODO should we support more functions which are not PPL builtin functions. E.g Spark builtin functions | ||
throw new UnsupportedOperationException(function.getFuncName() + " is not a builtin function of PPL"); | ||
} else { | ||
String name = BuiltinFunctionName.of(function.getFuncName()).get().name().toLowerCase(Locale.ROOT); | ||
return new UnresolvedFunction(seq(name), seq(args), false, empty(),false); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,163 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.flint.spark.ppl | ||
|
||
import org.junit.Assert.assertEquals | ||
import org.opensearch.flint.spark.ppl.PlaneUtils.plan | ||
import org.opensearch.sql.common.antlr.SyntaxCheckException | ||
import org.opensearch.sql.ppl.{CatalystPlanContext, CatalystQueryPlanVisitor} | ||
import org.opensearch.sql.ppl.utils.DataTypeTransformer.seq | ||
import org.scalatest.matchers.should.Matchers | ||
|
||
import org.apache.spark.SparkFunSuite | ||
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar} | ||
import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal} | ||
import org.apache.spark.sql.catalyst.plans.logical.{Filter, Project} | ||
|
||
class PPLLogicalPlanMathFunctionsTranslatorTestSuite | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. please add additional IT tests that use these commands with a more complex context of full fledge query use cases There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done |
||
extends SparkFunSuite | ||
with LogicalPlanTestUtils | ||
with Matchers { | ||
|
||
private val planTransformer = new CatalystQueryPlanVisitor() | ||
private val pplParser = new PPLSyntaxParser() | ||
|
||
test("test abs") { | ||
val context = new CatalystPlanContext | ||
val logPlan = planTransformer.visit(plan(pplParser, "source=t a = abs(b)", false), context) | ||
|
||
val table = UnresolvedRelation(Seq("t")) | ||
val filterExpr = EqualTo( | ||
UnresolvedAttribute("a"), | ||
UnresolvedFunction("abs", seq(UnresolvedAttribute("b")), isDistinct = false)) | ||
val filterPlan = Filter(filterExpr, table) | ||
val projectList = Seq(UnresolvedStar(None)) | ||
val expectedPlan = Project(projectList, filterPlan) | ||
assertEquals(expectedPlan, logPlan) | ||
} | ||
|
||
test("test ceil") { | ||
val context = new CatalystPlanContext | ||
val logPlan = planTransformer.visit(plan(pplParser, "source=t a = ceil(b)", false), context) | ||
|
||
val table = UnresolvedRelation(Seq("t")) | ||
val filterExpr = EqualTo( | ||
UnresolvedAttribute("a"), | ||
UnresolvedFunction("ceil", seq(UnresolvedAttribute("b")), isDistinct = false)) | ||
val filterPlan = Filter(filterExpr, table) | ||
val projectList = Seq(UnresolvedStar(None)) | ||
val expectedPlan = Project(projectList, filterPlan) | ||
assertEquals(expectedPlan, logPlan) | ||
} | ||
|
||
test("test floor") { | ||
val context = new CatalystPlanContext | ||
val logPlan = planTransformer.visit(plan(pplParser, "source=t a = floor(b)", false), context) | ||
|
||
val table = UnresolvedRelation(Seq("t")) | ||
val filterExpr = EqualTo( | ||
UnresolvedAttribute("a"), | ||
UnresolvedFunction("floor", seq(UnresolvedAttribute("b")), isDistinct = false)) | ||
val filterPlan = Filter(filterExpr, table) | ||
val projectList = Seq(UnresolvedStar(None)) | ||
val expectedPlan = Project(projectList, filterPlan) | ||
assertEquals(expectedPlan, logPlan) | ||
} | ||
|
||
test("test ln") { | ||
val context = new CatalystPlanContext | ||
val logPlan = planTransformer.visit(plan(pplParser, "source=t a = ln(b)", false), context) | ||
|
||
val table = UnresolvedRelation(Seq("t")) | ||
val filterExpr = EqualTo( | ||
UnresolvedAttribute("a"), | ||
UnresolvedFunction("ln", seq(UnresolvedAttribute("b")), isDistinct = false)) | ||
val filterPlan = Filter(filterExpr, table) | ||
val projectList = Seq(UnresolvedStar(None)) | ||
val expectedPlan = Project(projectList, filterPlan) | ||
assertEquals(expectedPlan, logPlan) | ||
} | ||
|
||
test("test mod") { | ||
val context = new CatalystPlanContext | ||
val logPlan = | ||
planTransformer.visit(plan(pplParser, "source=t a = mod(10, 4)", false), context) | ||
|
||
val table = UnresolvedRelation(Seq("t")) | ||
val filterExpr = EqualTo( | ||
UnresolvedAttribute("a"), | ||
UnresolvedFunction("mod", seq(Literal(10), Literal(4)), isDistinct = false)) | ||
val filterPlan = Filter(filterExpr, table) | ||
val projectList = Seq(UnresolvedStar(None)) | ||
val expectedPlan = Project(projectList, filterPlan) | ||
assertEquals(expectedPlan, logPlan) | ||
} | ||
|
||
test("test pow") { | ||
val context = new CatalystPlanContext | ||
val logPlan = planTransformer.visit(plan(pplParser, "source=t a = pow(2, 3)", false), context) | ||
|
||
val table = UnresolvedRelation(Seq("t")) | ||
val filterExpr = EqualTo( | ||
UnresolvedAttribute("a"), | ||
UnresolvedFunction("pow", seq(Literal(2), Literal(3)), isDistinct = false)) | ||
val filterPlan = Filter(filterExpr, table) | ||
val projectList = Seq(UnresolvedStar(None)) | ||
val expectedPlan = Project(projectList, filterPlan) | ||
assertEquals(expectedPlan, logPlan) | ||
} | ||
|
||
test("test sqrt") { | ||
val context = new CatalystPlanContext | ||
val logPlan = planTransformer.visit(plan(pplParser, "source=t a = sqrt(b)", false), context) | ||
|
||
val table = UnresolvedRelation(Seq("t")) | ||
val filterExpr = EqualTo( | ||
UnresolvedAttribute("a"), | ||
UnresolvedFunction("sqrt", seq(UnresolvedAttribute("b")), isDistinct = false)) | ||
val filterPlan = Filter(filterExpr, table) | ||
val projectList = Seq(UnresolvedStar(None)) | ||
val expectedPlan = Project(projectList, filterPlan) | ||
assertEquals(expectedPlan, logPlan) | ||
} | ||
|
||
test("test arithmetic: + - * / %") { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. please add a few more complex test for such operators There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done |
||
val context = new CatalystPlanContext | ||
val logPlan = | ||
planTransformer.visit( | ||
plan( | ||
pplParser, | ||
"source=t | where sqrt(pow(a, 2)) + sqrt(pow(a, 2)) / 1 - sqrt(pow(a, 2)) * 1 = sqrt(pow(a, 2)) % 1", | ||
false), | ||
context) | ||
val table = UnresolvedRelation(Seq("t")) | ||
// sqrt(pow(a, 2)) | ||
val sqrtPow = | ||
UnresolvedFunction( | ||
"sqrt", | ||
seq( | ||
UnresolvedFunction( | ||
"pow", | ||
seq(UnresolvedAttribute("a"), Literal(2)), | ||
isDistinct = false)), | ||
isDistinct = false) | ||
// sqrt(pow(a, 2)) / 1 | ||
val sqrtPowDivide = UnresolvedFunction("divide", seq(sqrtPow, Literal(1)), isDistinct = false) | ||
// sqrt(pow(a, 2)) * 1 | ||
val sqrtPowMultiply = | ||
UnresolvedFunction("multiply", seq(sqrtPow, Literal(1)), isDistinct = false) | ||
// sqrt(pow(a, 2)) % 1 | ||
val sqrtPowMod = UnresolvedFunction("modulus", seq(sqrtPow, Literal(1)), isDistinct = false) | ||
// sqrt(pow(a, 2)) + sqrt(pow(a, 2)) / 1 | ||
val add = UnresolvedFunction("add", seq(sqrtPow, sqrtPowDivide), isDistinct = false) | ||
val sub = UnresolvedFunction("subtract", seq(add, sqrtPowMultiply), isDistinct = false) | ||
val filterExpr = EqualTo(sub, sqrtPowMod) | ||
val filterPlan = Filter(filterExpr, table) | ||
val projectList = Seq(UnresolvedStar(None)) | ||
val expectedPlan = Project(projectList, filterPlan) | ||
assertEquals(expectedPlan, logPlan) | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Added
inExpr
back since it is required byposition
functions.