Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

[NSE-732]Adding new config to enable/disable complex data type support #777

Merged
merged 2 commits into from
Mar 22, 2022
Merged

[NSE-732]Adding new config to enable/disable complex data type support #777

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
26a3c12
Adding new config to enable/disable complex data type support
zhouyuan Mar 21, 2022
8eb22c7
remove sort fix
zhouyuan Mar 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions native-sql-engine/core/src/main/scala/com/intel/oap/GazellePluginConfig.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,10 @@ class GazellePluginConfig(conf: SQLConf) extends Logging {
.getConfString("spark.shuffle.manager", "sort")
.equals("org.apache.spark.shuffle.sort.ColumnarShuffleManager") && enableCpu

// enable experimental complex type support
val enableComplexType: Boolean =
conf.getConfString("spark.oap.sql.columnar.enableComplexType", "true").toBoolean

// for all perf turnings
// prefer to use columnar operators if set to true
val enablePreferColumnar: Boolean =
Expand Down
9 changes: 8 additions & 1 deletion ...gine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package org.apache.spark.sql.execution

import com.google.common.collect.Lists
import com.intel.oap.expression.{CodeGeneration, ColumnarExpression, ColumnarExpressionConverter, ConverterUtils}
import com.intel.oap.GazellePluginConfig
import com.intel.oap.vectorized.{ArrowColumnarBatchSerializer, ArrowWritableColumnVector, NativePartitioning}
import org.apache.arrow.gandiva.expression.TreeBuilder
import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema}
Expand Down Expand Up @@ -76,17 +77,23 @@ case class ColumnarShuffleExchangeExec(
override def output: Seq[Attribute] = child.output
buildCheck()


override def supportsColumnar: Boolean = true

override def stringArgs =
super.stringArgs ++ Iterator(s"[id=#$id]")
//super.stringArgs ++ Iterator(output.map(o => s"${o}#${o.dataType.simpleString}"))

def buildCheck(): Unit = {
val columnarConf: GazellePluginConfig = GazellePluginConfig.getSessionConf
// check input datatype
for (attr <- child.output) {
try {
ConverterUtils.createArrowField(attr)
if (!columnarConf.enableComplexType) {
ConverterUtils.checkIfTypeSupported(attr.dataType)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@zhixingheyi-tian can you please also check if this make sense to you? This helps to easily disable(fallback) on complex type

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make sense.

} else {
ConverterUtils.createArrowField(attr)
}
} catch {
case e: UnsupportedOperationException =>
throw new UnsupportedOperationException(
Expand Down