From be4bdb1d1b70ce1f1fa89cccc8c81e0e7c67f48c Mon Sep 17 00:00:00 2001 From: Utkarsh Date: Tue, 15 Oct 2024 10:57:38 -0500 Subject: [PATCH 1/2] fix --- .../sql/catalyst/expressions/Expression.scala | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala index 6a57ba2aaa569..194c5ac755aea 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala @@ -1347,9 +1347,22 @@ trait CommutativeExpression extends Expression { /** Collects adjacent commutative operations. */ private def gatherCommutative( e: Expression, - f: PartialFunction[CommutativeExpression, Seq[Expression]]): Seq[Expression] = e match { - case c: CommutativeExpression if f.isDefinedAt(c) => f(c).flatMap(gatherCommutative(_, f)) - case other => other.canonicalized :: Nil + f: PartialFunction[CommutativeExpression, Seq[Expression]]): Seq[Expression] = { + val resultBuffer = scala.collection.mutable.Buffer[Expression]() + val stack = scala.collection.mutable.Stack[Expression](e) + + // [SPARK-49977]: Use iterative approach to avoid creating many temporary List objects + // for deep expression trees through recursion. + while (stack.nonEmpty) { + val current = stack.pop() + current match { + case c: CommutativeExpression if f.isDefinedAt(c) => + stack.pushAll(f(c)) + case other => + resultBuffer += other.canonicalized + } + } + resultBuffer.toSeq } /** From fbea11f0c3f41e241ce939c038104df7b4206cef Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Thu, 17 Oct 2024 09:05:39 +0800 Subject: [PATCH 2/2] Update sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala Co-authored-by: Pat Sukprasert --- .../org/apache/spark/sql/catalyst/expressions/Expression.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala index 194c5ac755aea..bb32e518ec39a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala @@ -1354,8 +1354,7 @@ trait CommutativeExpression extends Expression { // [SPARK-49977]: Use iterative approach to avoid creating many temporary List objects // for deep expression trees through recursion. while (stack.nonEmpty) { - val current = stack.pop() - current match { + stack.pop() match { case c: CommutativeExpression if f.isDefinedAt(c) => stack.pushAll(f(c)) case other =>