diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarPlugin.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarPlugin.scala index f3b17923e..46ec91e19 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarPlugin.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarPlugin.scala @@ -35,7 +35,6 @@ import org.apache.spark.sql.internal.SQLConf case class ColumnarPreOverrides(conf: SparkConf) extends Rule[SparkPlan] { val columnarConf: ColumnarPluginConfig = ColumnarPluginConfig.getSessionConf var isSupportAdaptive: Boolean = true - val testing: Boolean = columnarConf.isTesting def replaceWithColumnarPlan(plan: SparkPlan): SparkPlan = plan match { case RowGuard(child: CustomShuffleReaderExec) => diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala index 830ceee4f..4e544ec81 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala @@ -35,11 +35,8 @@ class SingleLevelAggregateHashMapSuite extends DataFrameAggregateSuite with Befo .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -69,11 +66,8 @@ class TwoLevelAggregateHashMapSuite extends DataFrameAggregateSuite with BeforeA .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -105,11 +99,8 @@ class TwoLevelAggregateHashMapWithVectorizedMapSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproxCountDistinctForIntervalsQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproxCountDistinctForIntervalsQuerySuite.scala index 8875afff4..b7fbe1563 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproxCountDistinctForIntervalsQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproxCountDistinctForIntervalsQuerySuite.scala @@ -38,11 +38,8 @@ class ApproxCountDistinctForIntervalsQuerySuite extends QueryTest with SharedSpa .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala index 0345065d4..7c69c3af7 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala @@ -43,11 +43,8 @@ class ApproximatePercentileQuerySuite extends QueryTest with SharedSparkSession .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala index 01223cc96..597877ed1 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala @@ -58,11 +58,8 @@ class CachedTableSuite extends QueryTest with SQLTestUtils .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala index a44141692..f3b1d7fe3 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala @@ -47,11 +47,8 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala index 19dd2836b..2eb959a4c 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala @@ -35,11 +35,8 @@ class ComplexTypesSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala index cc67da762..98705d30d 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala @@ -39,11 +39,8 @@ class ConfigBehaviorSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CountMinSketchAggQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CountMinSketchAggQuerySuite.scala index e1c7d0422..a6f3ba0b5 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CountMinSketchAggQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CountMinSketchAggQuerySuite.scala @@ -37,11 +37,8 @@ class CountMinSketchAggQuerySuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala index e6abef015..a5b4e06cf 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala @@ -42,11 +42,8 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala index ccf86b4c4..7d91919d8 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -51,11 +51,8 @@ class DataFrameAggregateSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala index 496544878..e9d557297 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala @@ -43,11 +43,8 @@ class DataFrameComplexTypeSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala index 71a76d8fc..ac7563dff 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala @@ -51,11 +51,8 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameHintSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameHintSuite.scala index 2b7115a21..b46623ec0 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameHintSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameHintSuite.scala @@ -37,11 +37,8 @@ class DataFrameHintSuite extends AnalysisTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameImplicitsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameImplicitsSuite.scala index 00db39f8f..f654b5f74 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameImplicitsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameImplicitsSuite.scala @@ -34,11 +34,8 @@ class DataFrameImplicitsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala index 2d088e727..afd619d8f 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala @@ -48,11 +48,8 @@ class DataFrameJoinSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala index 9c2e970b0..7aa19abbd 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala @@ -38,11 +38,8 @@ class DataFrameNaFunctionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala index c0293d3c0..5163cd0af 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala @@ -40,11 +40,8 @@ class DataFramePivotSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameRangeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameRangeSuite.scala index 1e90cda60..c45eea7db 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameRangeSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameRangeSuite.scala @@ -41,11 +41,8 @@ class DataFrameRangeSuite extends QueryTest with SharedSparkSession with Eventua .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala index 9cfdd757c..c1c5d3405 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala @@ -37,11 +37,8 @@ class DataFrameSelfJoinSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala index 9a244a258..97e71368a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala @@ -41,11 +41,8 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala index 5026749f0..7c826f63a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala @@ -42,11 +42,8 @@ class DataFrameStatSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index 937adeeca..a4b265365 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -63,11 +63,8 @@ class DataFrameSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala index 4a210dd4f..be834d51a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala @@ -39,9 +39,6 @@ class DataFrameTimeWindowingSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") // .set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTungstenSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTungstenSuite.scala index 7fe8411b9..013d79e93 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTungstenSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTungstenSuite.scala @@ -42,11 +42,8 @@ class DataFrameTungstenSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala index d1b5a17e4..e489b173f 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala @@ -41,11 +41,8 @@ class DataFrameWindowFramesSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala index 1dd4cac23..61392bee6 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala @@ -49,11 +49,8 @@ class DataFrameWindowFunctionsSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala index 0d85e7b71..010d3ee5e 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala @@ -53,11 +53,8 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -123,7 +120,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("Append: basic append") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo") checkAnswer(spark.table("testcat.table_name"), Seq.empty) @@ -168,7 +165,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("Overwrite: overwrite by expression: true") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.sql( "CREATE TABLE testcat.table_name (id bigint, data string) USING foo PARTITIONED BY (id)") @@ -189,7 +186,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("Overwrite: overwrite by expression: id = 3") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.sql( "CREATE TABLE testcat.table_name (id bigint, data string) USING foo PARTITIONED BY (id)") @@ -236,7 +233,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("OverwritePartitions: overwrite conflicting partitions") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.sql( "CREATE TABLE testcat.table_name (id bigint, data string) USING foo PARTITIONED BY (id)") @@ -258,7 +255,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("OverwritePartitions: overwrite all rows if not partitioned") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo") checkAnswer(spark.table("testcat.table_name"), Seq.empty) @@ -304,7 +301,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("Create: basic behavior") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.table("source").writeTo("testcat.table_name").create() checkAnswer( @@ -321,7 +318,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("Create: with using") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.table("source").writeTo("testcat.table_name").using("foo").create() checkAnswer( @@ -338,7 +335,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("Create: with property") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.table("source").writeTo("testcat.table_name").tableProperty("prop", "value").create() checkAnswer( @@ -355,7 +352,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("Create: identity partitioned table") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.table("source").writeTo("testcat.table_name").partitionedBy($"id").create() checkAnswer( @@ -461,7 +458,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("Replace: basic behavior") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.sql( "CREATE TABLE testcat.table_name (id bigint, data string) USING foo PARTITIONED BY (id)") spark.sql("INSERT INTO TABLE testcat.table_name SELECT * FROM source") @@ -500,7 +497,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("Replace: partitioned table") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo") spark.sql("INSERT INTO TABLE testcat.table_name SELECT * FROM source") @@ -546,7 +543,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("CreateOrReplace: table does not exist") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.table("source2").writeTo("testcat.table_name").createOrReplace() checkAnswer( @@ -564,7 +561,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("CreateOrReplace: table exists") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { spark.sql( "CREATE TABLE testcat.table_name (id bigint, data string) USING foo PARTITIONED BY (id)") spark.sql("INSERT INTO TABLE testcat.table_name SELECT * FROM source") @@ -603,7 +600,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo } test("SPARK-30289 Create: partitioned by nested column") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val schema = new StructType().add("ts", new StructType() .add("created", TimestampType) .add("modified", TimestampType) diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala index 7d52dad8d..b4ff761f6 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala @@ -243,11 +243,8 @@ class DatasetAggregatorSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala index 97d7dd5b5..b2a213a51 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala @@ -44,11 +44,8 @@ class DatasetCacheSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala index 9ec47aa48..e5169f679 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala @@ -40,11 +40,8 @@ class DatasetOptimizationSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala index f854d7b3e..defede1aa 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala @@ -62,11 +62,8 @@ class DatasetPrimitiveSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala index a389d1e86..50054eae0 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala @@ -42,11 +42,8 @@ class DatasetSerializerRegistratorSuite extends QueryTest with SharedSparkSessio .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index 60d620bf1..665f52deb 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -67,11 +67,8 @@ class DatasetSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala index ba1a7a576..36490206b 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala @@ -46,11 +46,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala index 93d36c4d4..3882cc6fa 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala @@ -37,11 +37,8 @@ class DeprecatedAPISuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedDatasetAggregatorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedDatasetAggregatorSuite.scala index 3643c14b0..5d6caeeba 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedDatasetAggregatorSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedDatasetAggregatorSuite.scala @@ -37,11 +37,8 @@ class DeprecatedDatasetAggregatorSuite extends QueryTest with SharedSparkSession .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala index 1e31cc2f2..274d4e4e4 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala @@ -54,11 +54,8 @@ abstract class DynamicPartitionPruningSuiteBase .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala index e4a2041c1..f78e703e2 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala @@ -86,11 +86,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -381,11 +378,8 @@ class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuit .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala index a0cab89d9..294244f7b 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala @@ -78,11 +78,8 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala index 1409d5e85..199604464 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala @@ -61,11 +61,8 @@ class ExtraStrategiesSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala index dc3c377ff..bf5be90f4 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala @@ -58,11 +58,8 @@ class FileBasedDataSourceSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala index de371dcf8..972f27549 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala @@ -40,11 +40,8 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala index 1c1efc300..b385587e7 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala @@ -43,11 +43,8 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala index 8a44b82db..5a4f79d74 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala @@ -51,11 +51,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala index 0161e1b2a..87f3f72cc 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala @@ -42,11 +42,8 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala index e7290c2ca..5fcfd5411 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala @@ -45,11 +45,8 @@ class MathFunctionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala index ef6dfbfed..8c84e7dcf 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala @@ -39,11 +39,8 @@ abstract class MetadataCacheSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala index f81ccc4de..3a9b4e430 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala @@ -34,11 +34,8 @@ class MiscFunctionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala index 38c4e6582..da1c6492f 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala @@ -39,11 +39,8 @@ class ReplaceNullWithFalseInPredicateEndToEndSuite extends QueryTest with Shared .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/RowSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/RowSuite.scala index 4acba4f04..cc710cba6 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/RowSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/RowSuite.scala @@ -37,11 +37,8 @@ class RowSuite extends SparkFunSuite with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index b91681203..04ece5e78 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -58,11 +58,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala index 84bc90ea3..b23993c27 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala @@ -90,11 +90,8 @@ class ScalaReflectionRelationSuite extends SparkFunSuite with SharedSparkSession .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala index b24e797bf..dcc628223 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala @@ -54,11 +54,8 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index f52e8a6c4..c07b2734d 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala @@ -36,11 +36,8 @@ class StringFunctionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala index c8f3133c4..1f9ac3887 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala @@ -42,11 +42,8 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala index d13479512..a3fc574c0 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala @@ -38,11 +38,8 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCHQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCHQuerySuite.scala index 46a724678..172b531e5 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCHQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCHQuerySuite.scala @@ -37,11 +37,8 @@ class TPCHQuerySuite extends BenchmarkQueryTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala index 6c61c8d46..a34ea45a7 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala @@ -45,11 +45,8 @@ class TypedImperativeAggregateSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala index 64a50b82b..2b6baddc5 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala @@ -49,11 +49,8 @@ class UDFSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala index 526ff2873..fe6872211 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala @@ -138,11 +138,8 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/XPathFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/XPathFunctionsSuite.scala index 4efa0acc0..a6ee7abac 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/XPathFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/XPathFunctionsSuite.scala @@ -37,11 +37,8 @@ class XPathFunctionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala index 51031d048..75854bd3a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala @@ -38,11 +38,8 @@ trait AlterTableTests extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala index 1cd4a72e1..96e28a842 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala @@ -47,11 +47,8 @@ class DataSourceV2DataFrameSessionCatalogSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala index 168fd6deb..422cb9655 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala @@ -44,11 +44,8 @@ class DataSourceV2DataFrameSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -80,7 +77,7 @@ class DataSourceV2DataFrameSuite } test("insertInto: append across catalog") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" val t2 = "testcat2.db.tbl" withTable(t1, t2) { @@ -95,7 +92,7 @@ class DataSourceV2DataFrameSuite } test("saveAsTable: table doesn't exist => create table") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" withTable(t1) { val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") @@ -106,7 +103,7 @@ class DataSourceV2DataFrameSuite } ignore("saveAsTable: table exists => append by name") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING foo") @@ -125,7 +122,7 @@ class DataSourceV2DataFrameSuite } test("saveAsTable: table overwrite and table doesn't exist => create table") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" withTable(t1) { val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") @@ -136,7 +133,7 @@ class DataSourceV2DataFrameSuite } test("saveAsTable: table overwrite and table exists => replace table") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" withTable(t1) { sql(s"CREATE TABLE $t1 USING foo AS SELECT 'c', 'd'") @@ -148,7 +145,7 @@ class DataSourceV2DataFrameSuite } test("saveAsTable: ignore mode and table doesn't exist => create table") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" withTable(t1) { val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") @@ -159,7 +156,7 @@ class DataSourceV2DataFrameSuite } test("saveAsTable: ignore mode and table exists => do nothing") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" withTable(t1) { val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") @@ -171,7 +168,7 @@ class DataSourceV2DataFrameSuite } test("SPARK-29778: saveAsTable: append mode takes write options") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { var plan: LogicalPlan = null val listener = new QueryExecutionListener { override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = { @@ -208,7 +205,7 @@ class DataSourceV2DataFrameSuite } test("Cannot write data with intervals to v2") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { withTable("testcat.table_name") { val testCatalog = spark.sessionState.catalogManager.catalog("testcat").asTableCatalog testCatalog.createTable( diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala index dbfac3b54..a69246aaa 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala @@ -37,11 +37,8 @@ class DataSourceV2SQLSessionCatalogSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index fbe94ee2e..2c4956ce0 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -50,11 +50,8 @@ class DataSourceV2SQLSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -744,7 +741,7 @@ class DataSourceV2SQLSuite } test("Relation: basic") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" withTable(t1) { sql(s"CREATE TABLE $t1 USING foo AS SELECT id, data FROM source") @@ -755,7 +752,7 @@ class DataSourceV2SQLSuite } test("Relation: SparkSession.table()") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" withTable(t1) { sql(s"CREATE TABLE $t1 USING foo AS SELECT id, data FROM source") @@ -765,7 +762,7 @@ class DataSourceV2SQLSuite } test("Relation: CTE") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" withTable(t1) { sql(s"CREATE TABLE $t1 USING foo AS SELECT id, data FROM source") @@ -792,7 +789,7 @@ class DataSourceV2SQLSuite } test("Relation: join tables in 2 catalogs") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" val t2 = "testcat2.v2tbl" withTable(t1, t2) { @@ -814,7 +811,7 @@ class DataSourceV2SQLSuite } test("qualified column names for v2 tables") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t = "testcat.ns1.ns2.tbl" withTable(t) { sql(s"CREATE TABLE $t (id bigint, point struct) USING foo") @@ -845,7 +842,7 @@ class DataSourceV2SQLSuite } test("qualified column names for v1 tables") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { Seq(true, false).foreach { useV1Table => val format = if (useV1Table) "json" else v2Format if (useV1Table) { @@ -871,7 +868,7 @@ class DataSourceV2SQLSuite } test("InsertInto: append - across catalog") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = "testcat.ns1.ns2.tbl" val t2 = "testcat2.db.tbl" withTable(t1, t2) { @@ -1763,7 +1760,7 @@ class DataSourceV2SQLSuite } test("DeleteFrom: basic - delete with where clause") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t = "testcat.ns1.ns2.tbl" withTable(t) { sql(s"CREATE TABLE $t (id bigint, data string, p int) USING foo PARTITIONED BY (id, p)") @@ -1776,7 +1773,7 @@ class DataSourceV2SQLSuite } test("DeleteFrom: delete from aliased target table") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t = "testcat.ns1.ns2.tbl" withTable(t) { sql(s"CREATE TABLE $t (id bigint, data string, p int) USING foo PARTITIONED BY (id, p)") @@ -1789,7 +1786,7 @@ class DataSourceV2SQLSuite } test("DeleteFrom: normalize attribute names") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t = "testcat.ns1.ns2.tbl" withTable(t) { sql(s"CREATE TABLE $t (id bigint, data string, p int) USING foo PARTITIONED BY (id, p)") @@ -1802,7 +1799,7 @@ class DataSourceV2SQLSuite } test("DeleteFrom: fail if has subquery") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t = "testcat.ns1.ns2.tbl" withTable(t) { sql(s"CREATE TABLE $t (id bigint, data string, p int) USING foo PARTITIONED BY (id, p)") @@ -2367,7 +2364,7 @@ class DataSourceV2SQLSuite } test("SPARK-30094: current namespace is used during table resolution") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { // unset this config to use the default v2 session catalog. spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) @@ -2384,7 +2381,7 @@ class DataSourceV2SQLSuite } test("SPARK-30284: CREATE VIEW should track the current catalog and namespace") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { // unset this config to use the default v2 session catalog. spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) val sessionCatalogName = CatalogManager.SESSION_CATALOG_NAME @@ -2498,7 +2495,7 @@ class DataSourceV2SQLSuite } test("SPARK-31015: star expression should work for qualified column names for v2 tables") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t = "testcat.ns1.ns2.tbl" withTable(t) { sql(s"CREATE TABLE $t (id bigint, name string) USING foo") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala index 8d6d91a62..fb34c63db 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala @@ -57,11 +57,8 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala index b29e6fa61..cd95269ea 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala @@ -98,11 +98,8 @@ class FileDataSourceV2FallBackSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala index 52fa67998..40dc18660 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala @@ -53,11 +53,8 @@ abstract class InsertIntoTests( .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -70,7 +67,7 @@ abstract class InsertIntoTests( protected def doInsert(tableName: String, insert: DataFrame, mode: SaveMode = null): Unit test("insertInto: append") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") @@ -80,7 +77,7 @@ abstract class InsertIntoTests( } test("insertInto: append by position") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") @@ -92,7 +89,7 @@ abstract class InsertIntoTests( } test("insertInto: append partitioned table") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") @@ -104,7 +101,7 @@ abstract class InsertIntoTests( } test("insertInto: overwrite non-partitioned table") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") @@ -116,7 +113,7 @@ abstract class InsertIntoTests( } test("insertInto: overwrite partitioned table in static mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = s"${catalogAndNamespace}tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") @@ -132,7 +129,7 @@ abstract class InsertIntoTests( test("insertInto: overwrite partitioned table in static mode by position") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = s"${catalogAndNamespace}tbl" withTable(t1) { @@ -179,7 +176,7 @@ abstract class InsertIntoTests( } dynamicOverwriteTest("insertInto: overwrite partitioned table in dynamic mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") @@ -195,7 +192,7 @@ abstract class InsertIntoTests( } dynamicOverwriteTest("insertInto: overwrite partitioned table in dynamic mode by position") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") @@ -277,7 +274,7 @@ trait InsertIntoSQLOnlyTests } test("InsertInto: append to partitioned table - static clause") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") @@ -320,7 +317,7 @@ trait InsertIntoSQLOnlyTests } test("InsertInto: overwrite - dynamic clause - static mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => @@ -337,7 +334,7 @@ trait InsertIntoSQLOnlyTests } dynamicOverwriteTest("InsertInto: overwrite - dynamic clause - dynamic mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") @@ -353,7 +350,7 @@ trait InsertIntoSQLOnlyTests } test("InsertInto: overwrite - missing clause - static mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => @@ -370,7 +367,7 @@ trait InsertIntoSQLOnlyTests } dynamicOverwriteTest("InsertInto: overwrite - missing clause - dynamic mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") @@ -386,7 +383,7 @@ trait InsertIntoSQLOnlyTests } test("InsertInto: overwrite - static clause") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p1 int) " + @@ -403,7 +400,7 @@ trait InsertIntoSQLOnlyTests } test("InsertInto: overwrite - mixed clause - static mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => @@ -421,7 +418,7 @@ trait InsertIntoSQLOnlyTests } test("InsertInto: overwrite - mixed clause reordered - static mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => @@ -439,7 +436,7 @@ trait InsertIntoSQLOnlyTests } test("InsertInto: overwrite - implicit dynamic partition - static mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => @@ -457,7 +454,7 @@ trait InsertIntoSQLOnlyTests } dynamicOverwriteTest("InsertInto: overwrite - mixed clause - dynamic mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + @@ -474,7 +471,7 @@ trait InsertIntoSQLOnlyTests } dynamicOverwriteTest("InsertInto: overwrite - mixed clause reordered - dynamic mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + @@ -491,7 +488,7 @@ trait InsertIntoSQLOnlyTests } dynamicOverwriteTest("InsertInto: overwrite - implicit dynamic partition - dynamic mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + @@ -508,7 +505,7 @@ trait InsertIntoSQLOnlyTests } test("InsertInto: overwrite - multiple static partitions - dynamic mode") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { // Since all partitions are provided statically, this should be supported by everyone withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) { val t1 = s"${catalogAndNamespace}tbl" @@ -528,7 +525,7 @@ trait InsertIntoSQLOnlyTests } test("do not double insert on INSERT INTO collect()") { - withSQLConf("spark.oap.sql.columnar.testing" -> "true") { + withSQLConf("spark.oap.sql.columnar.batchscan" -> "false") { val t1 = s"${catalogAndNamespace}tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala index b1234ddc6..de5bb259f 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala @@ -50,11 +50,8 @@ class SupportsCatalogOptionsSuite extends QueryTest with SharedSparkSession with .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala index 582e61a2b..75c3e645b 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala @@ -49,11 +49,8 @@ class TableCapabilityCheckSuite extends AnalysisSuite with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala index efcf834eb..68b2032bd 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala @@ -48,11 +48,8 @@ abstract class V1ReadFallbackSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala index adbd05e78..4f45eb0f1 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala @@ -50,11 +50,8 @@ class V1WriteFallbackSuite extends QueryTest with SharedSparkSession with Before .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -161,11 +158,8 @@ class V1WriteFallbackSessionCatalogSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala index 71a2af583..d53f06c86 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala @@ -45,11 +45,8 @@ class V2CommandsCaseSensitivitySuite extends SharedSparkSession with AnalysisTes .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/AggregatingAccumulatorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/AggregatingAccumulatorSuite.scala index c17d3d85d..8a56a2407 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/AggregatingAccumulatorSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/AggregatingAccumulatorSuite.scala @@ -47,11 +47,8 @@ class AggregatingAccumulatorSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/BroadcastExchangeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/BroadcastExchangeSuite.scala index 906243075..4b80273f9 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/BroadcastExchangeSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/BroadcastExchangeSuite.scala @@ -45,11 +45,8 @@ class BroadcastExchangeSuite extends SparkPlanTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala index 633942c1d..16938e52a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala @@ -83,11 +83,8 @@ class DataSourceScanExecRedactionSuite extends DataSourceScanRedactionTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -154,11 +151,8 @@ class DataSourceV2ScanExecRedactionSuite extends DataSourceScanRedactionTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala index e21b5767f..93cdbf7fd 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala @@ -57,11 +57,8 @@ class ExchangeSuite extends SparkPlanTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala index 9a37cdeae..f89ba48bc 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala @@ -40,11 +40,8 @@ class GlobalTempViewSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala index e53d362d1..f1c7294d2 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala @@ -37,11 +37,8 @@ class HiveResultSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala index 2f964717c..89f55b75a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala @@ -40,11 +40,8 @@ class OptimizeMetadataOnlyQuerySuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala index 0218a9940..8609b4f13 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala @@ -50,11 +50,8 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala index fdfd1f71f..46d374191 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala @@ -48,11 +48,8 @@ class QueryExecutionSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala index 313b79da0..d551b0d63 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala @@ -35,11 +35,8 @@ class QueryPlanningTrackerEndToEndSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala index fb4429aec..267e27630 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala @@ -36,11 +36,8 @@ class SimpleSQLViewSuite extends SQLViewSuite with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala index 8187100f7..6885a81c1 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala @@ -44,11 +44,8 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala index c861418b6..a9ae4c719 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala @@ -44,11 +44,8 @@ class SameResultSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala index c5b2b8cec..a0006cc18 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala @@ -43,15 +43,11 @@ class SortSuite extends SparkPlanTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") - .set("spark.sql.columnar.sort", "true") .set("spark.sql.columnar.nanCheck", "true") test("basic sorting using ExternalSort") { diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala index 720c5ec06..4c658ed1d 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala @@ -36,11 +36,8 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala index 0b0bbddd2..1bff3b91e 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala @@ -37,11 +37,8 @@ class SparkPlannerSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala index 3153d5f54..6cb38dac7 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala @@ -40,11 +40,8 @@ class TakeOrderedAndProjectSuite extends SparkPlanTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala index b4d64d826..bdb8fdf5e 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala @@ -58,11 +58,8 @@ class UnsafeFixedWidthAggregationMapSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala index 46c10a45b..f30e49f28 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala @@ -47,11 +47,8 @@ class UnsafeKVExternalSorterSuite extends SparkFunSuite with SharedSparkSession .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala index d951ecd3a..09af5997d 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala @@ -47,11 +47,8 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala index a9035a047..5f683275a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala @@ -56,11 +56,8 @@ class AdaptiveQueryExecSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala index ced175a0d..cb741b66b 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala @@ -52,11 +52,8 @@ class ArrowConvertersSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala index 0b8c5cc87..7c4d39c0e 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala @@ -48,11 +48,8 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala index 5875d5c73..0a80b8589 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala @@ -38,11 +38,8 @@ class PartitionBatchPruningSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala index 13e7fe75a..3cd350fb2 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala @@ -54,11 +54,8 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 1a17829ae..ecadd0b02 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -53,11 +53,8 @@ class InMemoryCatalogedDDLSuite extends DDLSuite with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala index a9e5655d9..040fa755a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala @@ -38,11 +38,8 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala index e28717d1b..868d3c140 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala @@ -40,11 +40,8 @@ class FileFormatWriterSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala index e7ac35b6c..27fc16219 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala @@ -45,11 +45,8 @@ class FileIndexSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala index dda366ece..266fe4483 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala @@ -53,11 +53,8 @@ class FileSourceStrategySuite extends QueryTest with SharedSparkSession with Pre .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala index e923a5381..dfffb2641 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala @@ -39,11 +39,8 @@ class HadoopFileLinesReaderSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala index 01db34930..e36c951da 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala @@ -72,11 +72,8 @@ class CSVReadSchemaSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -105,11 +102,8 @@ class HeaderCSVReadSchemaSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -141,11 +135,8 @@ class JsonReadSchemaSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -173,11 +164,8 @@ class OrcReadSchemaSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -218,11 +206,8 @@ class VectorizedOrcReadSchemaSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -264,11 +249,8 @@ class MergedOrcReadSchemaSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -296,11 +278,8 @@ class ParquetReadSchemaSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -339,11 +318,8 @@ class VectorizedParquetReadSchemaSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -382,11 +358,8 @@ class MergedParquetReadSchemaSuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala index ec3a146d4..702a84fc0 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala @@ -42,11 +42,8 @@ class RowDataSourceStrategySuite extends SharedSparkSession with BeforeAndAfter .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala index 779af671b..50fa9cb91 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala @@ -34,11 +34,8 @@ class SaveIntoDataSourceCommandSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala index 128bb551d..73783ac05 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala @@ -50,11 +50,8 @@ class BinaryFileFormatSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala index d4c7428f8..31768dab3 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala @@ -2355,9 +2355,6 @@ class CSVv1Suite extends CSVSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") // .set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") @@ -2379,11 +2376,8 @@ class CSVv2Suite extends CSVSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -2403,11 +2397,8 @@ class CSVLegacyTimeParserSuite extends CSVSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala index 51535743c..8ecd8eba2 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala @@ -39,11 +39,8 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala index 06ba9bd74..869b90dd1 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala @@ -61,11 +61,8 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopStreamSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopStreamSuite.scala index 5c2c7712a..b037b8b5c 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopStreamSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopStreamSuite.scala @@ -36,11 +36,8 @@ class NoopStreamSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopSuite.scala index 4cab7c81c..54cadef26 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopSuite.scala @@ -34,11 +34,8 @@ class NoopSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala index 89bddd265..16ec57636 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala @@ -39,11 +39,8 @@ class OrcColumnarBatchReaderSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcPartitionDiscoverySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcPartitionDiscoverySuite.scala index 4247853a6..fd225ad89 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcPartitionDiscoverySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcPartitionDiscoverySuite.scala @@ -180,11 +180,8 @@ class OrcPartitionDiscoverySuite extends OrcPartitionDiscoveryTest with SharedSp .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -289,11 +286,8 @@ class OrcV1PartitionDiscoverySuite extends OrcPartitionDiscoveryTest with Shared .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala index d8bfb1f26..29ad8bb36 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala @@ -664,11 +664,8 @@ abstract class OrcQuerySuite extends OrcQueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala index a7087316f..ddb8d4315 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala @@ -554,11 +554,8 @@ class OrcSourceSuite extends OrcSuite with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1FilterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1FilterSuite.scala index d99dba987..a1b9d6f67 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1FilterSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1FilterSuite.scala @@ -40,11 +40,8 @@ class OrcV1FilterSuite extends OrcFilterSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1SchemaPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1SchemaPruningSuite.scala index c1e047ef7..bc36478f1 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1SchemaPruningSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1SchemaPruningSuite.scala @@ -37,11 +37,8 @@ class OrcV1SchemaPruningSuite extends SchemaPruningSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala index 5b5ca3887..e44bf7471 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala @@ -41,11 +41,8 @@ class OrcV2SchemaPruningSuite extends SchemaPruningSuite with AdaptiveSparkPlanH .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala index 25c9d2617..242146b81 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala @@ -45,11 +45,8 @@ class ParquetAvroCompatibilitySuite extends ParquetCompatibilityTest with Shared .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompressionCodecPrecedenceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompressionCodecPrecedenceSuite.scala index 6d4587667..085fe1f58 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompressionCodecPrecedenceSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompressionCodecPrecedenceSuite.scala @@ -39,11 +39,8 @@ class ParquetCompressionCodecPrecedenceSuite extends ParquetTest with SharedSpar .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala index 5002194cf..0c34333c3 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala @@ -37,11 +37,8 @@ class ParquetEncodingSuite extends ParquetCompatibilityTest with SharedSparkSess .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala index c4098f6ea..9020f4620 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala @@ -36,11 +36,8 @@ class ParquetFileFormatSuite extends QueryTest with ParquetTest with SharedSpark .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala index 20a7ac282..5f12e13cd 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala @@ -75,11 +75,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala index 708c04c4a..35cb3a137 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala @@ -90,11 +90,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala index 1cd1ca0ba..40444b724 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala @@ -44,11 +44,8 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala index acef87259..3b543c2ac 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala @@ -66,11 +66,8 @@ abstract class ParquetPartitionDiscoverySuite .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetProtobufCompatibilitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetProtobufCompatibilitySuite.scala index 5590d8fb4..5cec3db6b 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetProtobufCompatibilitySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetProtobufCompatibilitySuite.scala @@ -34,11 +34,8 @@ class ParquetProtobufCompatibilitySuite extends ParquetCompatibilityTest with Sh .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala index 9b7202056..24ff217fd 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala @@ -54,11 +54,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala index c2b7a7074..46935aec2 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala @@ -46,11 +46,8 @@ class ParquetV1SchemaPruningSuite extends ParquetSchemaPruningSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -71,11 +68,8 @@ class ParquetV2SchemaPruningSuite extends ParquetSchemaPruningSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala index b0d2834b5..7e560780b 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala @@ -131,11 +131,8 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -381,11 +378,8 @@ class ParquetSchemaSuite extends ParquetSchemaTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetThriftCompatibilitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetThriftCompatibilitySuite.scala index a101c3d1f..15c2b3909 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetThriftCompatibilitySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetThriftCompatibilitySuite.scala @@ -35,11 +35,8 @@ class ParquetThriftCompatibilitySuite extends ParquetCompatibilityTest with Shar .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala index 268de3e3d..bcdf38254 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala @@ -45,11 +45,8 @@ abstract class TextSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala index ecccad940..db9273644 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala @@ -118,11 +118,8 @@ class WholeTextFileV1Suite extends WholeTextFileSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -143,11 +140,8 @@ class WholeTextFileV2Suite extends WholeTextFileSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala index 1ec130671..9562d69e2 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala @@ -63,11 +63,8 @@ class FileTableSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala index e198c7e0b..5e00deb14 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala @@ -65,11 +65,8 @@ class V2SessionCatalogTableSuite extends V2SessionCatalogBaseSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -768,11 +765,8 @@ class V2SessionCatalogNamespaceSuite extends V2SessionCatalogBaseSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala index 29132f7a9..61e9f5919 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala @@ -42,11 +42,8 @@ class ExistenceJoinSuite extends SparkPlanTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "100m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala index 286a74614..e24dabe94 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala @@ -48,11 +48,8 @@ class HashedRelationSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala index eda607b76..c123f7868 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala @@ -44,11 +44,8 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "1g") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala index 44b073120..3fffedee1 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala @@ -42,11 +42,8 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala index e260268e0..798a44f41 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala @@ -51,11 +51,8 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala index 6341cea07..b636c7c13 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala @@ -46,11 +46,8 @@ class BatchEvalPythonExecSuite extends SparkPlanTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala index 29a2a545e..6c593a4f6 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala @@ -39,11 +39,8 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala index 226d0324e..6d78cea15 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala @@ -38,11 +38,8 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManagerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManagerSuite.scala index 7deaad88b..dc7e2cb86 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManagerSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManagerSuite.scala @@ -117,11 +117,8 @@ class CheckpointFileManagerSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala index da67b8f86..3bd04285f 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala @@ -39,11 +39,8 @@ class CompactibleFileStreamLogSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala index e8cad37db..bd05615e9 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala @@ -40,11 +40,8 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala index 93f8534ca..f22627462 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala @@ -42,11 +42,8 @@ class HDFSMetadataLogSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala index 83c808955..92b5601ad 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala @@ -43,11 +43,8 @@ class MemorySinkSuite extends StreamTest with BeforeAndAfter { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala index 4a4ebde62..37f2942e9 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala @@ -37,11 +37,8 @@ class MicroBatchExecutionSuite extends StreamTest with BeforeAndAfter { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala index e240a6d3c..c45c80554 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala @@ -37,11 +37,8 @@ class OffsetSeqLogSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamMetadataSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamMetadataSuite.scala index dd5a9a0bb..01ac7f129 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamMetadataSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamMetadataSuite.scala @@ -38,11 +38,8 @@ class StreamMetadataSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala index 86e90449b..fb71ae088 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala @@ -40,11 +40,8 @@ class ContinuousShuffleSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupportSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupportSuite.scala index 33b6f8434..6ceba71e2 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupportSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupportSuite.scala @@ -37,11 +37,8 @@ class ConsoleWriteSupportSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSinkSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSinkSuite.scala index 664345603..e59667217 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSinkSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSinkSuite.scala @@ -42,11 +42,8 @@ class ForeachBatchSinkSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala index d5ac3becd..00b323066 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala @@ -43,11 +43,8 @@ class ForeachWriterSuite extends StreamTest with SharedSparkSession with BeforeA .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProviderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProviderSuite.scala index 8d41ed842..4854a74f1 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProviderSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProviderSuite.scala @@ -50,11 +50,8 @@ class RateStreamProviderSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/TextSocketStreamSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/TextSocketStreamSuite.scala index c8a85f881..45631e0e9 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/TextSocketStreamSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/TextSocketStreamSuite.scala @@ -54,11 +54,8 @@ class TextSocketStreamSuite extends StreamTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala index 332d12399..8f3997b96 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala @@ -48,11 +48,8 @@ class AllExecutionsPageSuite extends SharedSparkSession with BeforeAndAfter { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala index a055166cd..3f3824ad5 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala @@ -64,11 +64,8 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala index 5049a1feb..aa08d9553 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala @@ -38,11 +38,8 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala index 930bc6835..139e5b7d9 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala @@ -48,11 +48,8 @@ class CatalogSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala index 351a33760..e926298df 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala @@ -38,11 +38,8 @@ class DeprecatedCreateExternalTableSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala index f39167eb3..d44326d4b 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala @@ -40,11 +40,8 @@ class SQLConfSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala index 9a0d81fc1..1565bfd10 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala @@ -42,11 +42,8 @@ class SharedStateSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 5013324fa..b4cc609aa 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -54,11 +54,8 @@ class JDBCSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala index e3b4895c4..e8cb0479e 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala @@ -46,11 +46,8 @@ class JDBCWriteSuite extends SharedSparkSession with BeforeAndAfter { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala index d84561ecd..204d3b1f6 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala @@ -53,11 +53,8 @@ class BucketedReadWithoutHiveSupportSuite extends BucketedReadSuite with SharedS .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala index a6e44170a..26f386111 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala @@ -44,11 +44,8 @@ class BucketedWriteWithoutHiveSupportSuite extends BucketedWriteSuite with Share .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala index 2ad5a41da..2b562450f 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala @@ -42,11 +42,8 @@ class CreateTableAsSelectSuite extends DataSourceTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DDLSourceLoadSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DDLSourceLoadSuite.scala index 4a19ed3f9..6dd276d12 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DDLSourceLoadSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DDLSourceLoadSuite.scala @@ -37,11 +37,8 @@ class DDLSourceLoadSuite extends DataSourceTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ExternalCommandRunnerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ExternalCommandRunnerSuite.scala index 003ca454f..285ac263f 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ExternalCommandRunnerSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ExternalCommandRunnerSuite.scala @@ -38,11 +38,8 @@ class ExternalCommandRunnerSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala index ae388a8c7..edcf0a078 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala @@ -146,11 +146,8 @@ class FilteredScanSuite extends DataSourceTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala index c9986cf9d..ec1e36917 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala @@ -66,11 +66,8 @@ class InsertSuite extends DataSourceTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala index 561083b62..de50e0b75 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala @@ -56,11 +56,8 @@ class PartitionedWriteSuite extends QueryTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala index baf161418..b2ff6964e 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala @@ -78,11 +78,8 @@ class PathOptionSuite extends DataSourceTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala index a1a317910..5b5881326 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala @@ -68,11 +68,8 @@ class PrunedScanSuite extends DataSourceTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala index 12b678f92..816872a66 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala @@ -36,11 +36,8 @@ class ResolvedDataSourceSuite extends SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala index 22e8608cd..455074ee9 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala @@ -43,11 +43,8 @@ class SaveLoadSuite extends DataSourceTest with SharedSparkSession with BeforeAn .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala index 57af31250..070322a9a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala @@ -123,11 +123,8 @@ class TableScanSuite extends DataSourceTest with SharedSparkSession { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/DeprecatedStreamingAggregationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/DeprecatedStreamingAggregationSuite.scala index bdad6f8a6..be6603084 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/DeprecatedStreamingAggregationSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/DeprecatedStreamingAggregationSuite.scala @@ -41,11 +41,8 @@ class DeprecatedStreamingAggregationSuite extends StateStoreMetricsTest with Ass .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala index a55e23988..75eaefcc1 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala @@ -52,11 +52,8 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala index 1c42e356a..85e3841d0 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala @@ -660,11 +660,8 @@ class FileStreamSinkV2Suite extends FileStreamSinkSuite { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala index 50e4f53cd..3a039fb75 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala @@ -236,11 +236,8 @@ class FileStreamSourceSuite extends FileStreamSourceTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -1975,11 +1972,8 @@ class FileStreamSourceStressTestSuite extends FileStreamSourceTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamStressSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamStressSuite.scala index f14bd8cfc..83cfbb961 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamStressSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamStressSuite.scala @@ -51,11 +51,8 @@ class FileStreamStressSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala index 64ec7a014..46fd4ffaa 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala @@ -62,11 +62,8 @@ class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala index 7eb0c1620..5ab2027f9 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala @@ -34,11 +34,8 @@ class MemorySourceStressSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala index 693a34b89..78c6435ce 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala @@ -63,11 +63,8 @@ class StreamSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala index 3aa0e84a3..e4ad851ac 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala @@ -61,11 +61,8 @@ class StreamingAggregationSuite extends StateStoreMetricsTest with Assertions { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala index 0f335fafa..b9807fd52 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala @@ -41,11 +41,8 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala index 419a52038..17b2aad0d 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala @@ -52,11 +52,8 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") @@ -515,11 +512,8 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala index 613058f6d..608f46d55 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala @@ -50,11 +50,8 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenersConfSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenersConfSuite.scala index 5fd8e7045..9eecb4842 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenersConfSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenersConfSuite.scala @@ -40,11 +40,8 @@ class StreamingQueryListenersConfSuite extends StreamTest with BeforeAndAfter { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryManagerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryManagerSuite.scala index 2841dd19f..80a709da9 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryManagerSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryManagerSuite.scala @@ -50,11 +50,8 @@ class StreamingQueryManagerSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala index 5707b6c94..b2b7d0b55 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala @@ -50,11 +50,8 @@ class StreamingQueryStatusAndProgressSuite extends StreamTest with Eventually { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala index 81f3de23f..8ae5c619e 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala @@ -59,11 +59,8 @@ class StreamingQuerySuite extends StreamTest with BeforeAndAfter with Logging wi .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSymmetricHashJoinHelperSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSymmetricHashJoinHelperSuite.scala index e8493a89f..57e553dbb 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSymmetricHashJoinHelperSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSymmetricHashJoinHelperSuite.scala @@ -38,11 +38,8 @@ class StreamingSymmetricHashJoinHelperSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala index 3d5c55911..bd3170ced 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala @@ -38,11 +38,8 @@ class ContinuousAggregationSuite extends ContinuousSuiteBase { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueuedDataReaderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueuedDataReaderSuite.scala index a6500fa75..31cbc8684 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueuedDataReaderSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueuedDataReaderSuite.scala @@ -44,11 +44,8 @@ class ContinuousQueuedDataReaderSuite extends StreamTest with MockitoSugar { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala index c356d5854..4f87ea4eb 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala @@ -44,11 +44,8 @@ class ContinuousSuiteBase extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala index 7a8fca276..f49fdec0b 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala @@ -252,11 +252,8 @@ class StreamingDataSourceV2Suite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala index 15a21a5ae..e3cace2fb 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala @@ -120,11 +120,8 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala index 6bd4934be..aadeceb2a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala @@ -41,11 +41,8 @@ class StreamingQueryPageSuite extends SharedSparkSession with BeforeAndAfter { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala index a941d2669..2348fe942 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala @@ -37,11 +37,8 @@ class StreamingQueryStatusListenerSuite extends StreamTest { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala index c27080d88..9afc298fb 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala @@ -154,11 +154,8 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala index 76275694e..530a0e9b4 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala @@ -39,11 +39,8 @@ class GenericFlatSpecSuite extends FlatSpec with SharedSparkSessionBase { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala index d6a1b8b7f..f5f77e6f6 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala @@ -39,11 +39,8 @@ class GenericFunSpecSuite extends FunSpec with SharedSparkSessionBase { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala index 12d18091e..432b114b3 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala @@ -39,11 +39,8 @@ class GenericWordSpecSuite extends WordSpec with SharedSparkSessionBase { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameAggregateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameAggregateSuite.scala index 51fa0279b..288d0e206 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameAggregateSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameAggregateSuite.scala @@ -51,11 +51,8 @@ class TravisDataFrameAggregateSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameJoinSuite.scala index ffcacfa4d..cc684fc0a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameJoinSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameJoinSuite.scala @@ -48,11 +48,8 @@ class TravisDataFrameJoinSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true") diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala index b6e037f84..d2c565b6a 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala @@ -49,11 +49,8 @@ class DataFrameCallbackSuite extends QueryTest .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "50m") .set("spark.sql.join.preferSortMergeJoin", "false") - .set("spark.sql.columnar.codegen.hashAggregate", "false") - .set("spark.oap.sql.columnar.wholestagecodegen", "true") - .set("spark.sql.columnar.window", "true") .set("spark.unsafe.exceptionOnMemoryLeak", "false") - //.set("spark.sql.columnar.tmp_dir", "/codegen/nativesql/") + //.set("spark.oap.sql.columnar.tmp_dir", "/codegen/nativesql/") .set("spark.sql.columnar.sort.broadcastJoin", "true") .set("spark.oap.sql.columnar.preferColumnar", "true") .set("spark.oap.sql.columnar.sortmergejoin", "true")