Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

Commit

Permalink
Avoid sharing buffers in output
Browse files Browse the repository at this point in the history
  • Loading branch information
zhztheplayer committed Mar 24, 2021
1 parent a4e4ae5 commit 7dca7d2
Showing 1 changed file with 18 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import org.apache.arrow.util.AutoCloseables
import org.apache.arrow.vector.ValueVector

import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import scala.util.control.Breaks._

class ColumnarConditionProjector(
Expand Down Expand Up @@ -463,10 +464,24 @@ object ColumnarConditionProjector extends Logging {
class FieldOptimizedProjector(projectionSchema: Schema, resultSchema: Schema,
exprs: java.util.List[ExpressionTree]) extends ProjectorWrapper {

val fieldExprs = exprs.asScala.zipWithIndex.filter {
case (expr, _) =>
val fieldExprs = ListBuffer[(ExpressionTree, Int)]()
val fieldExprNames = new util.HashSet[String]()

/**
* nonFieldExprs may include fields that are already appeared in projection list.
* To avoid sharing same buffers over output columns.
*/
val nonFieldExprs = ListBuffer[(ExpressionTree, Int)]()

exprs.asScala.zipWithIndex.foreach {
case (expr, i) =>
val root = getRoot(expr)
fieldClazz.isInstance(root)
if (fieldClazz.isInstance(root) && !fieldExprNames.contains(getField(root).getName)) {
fieldExprs.append((expr, i))
fieldExprNames.add(getField(root).getName)
} else {
nonFieldExprs.append((expr, i))
}
}

val fieldResultSchema = new Schema(
Expand All @@ -475,12 +490,6 @@ object ColumnarConditionProjector extends Logging {
resultSchema.getFields.get(i)
}.asJava)

val nonFieldExprs = exprs.asScala.zipWithIndex.filter {
case (expr, _) =>
val root = getRoot(expr)
!fieldClazz.isInstance(root)
}

val nonFieldResultSchema = new Schema(
nonFieldExprs.map {
case (_, i) =>
Expand Down

0 comments on commit 7dca7d2

Please sign in to comment.