From c319ee9cea78544406e30dd36cc603fd0a0283db Mon Sep 17 00:00:00 2001 From: Raymond Xu <2701446+xushiyan@users.noreply.github.com> Date: Fri, 6 May 2022 05:52:06 -0700 Subject: [PATCH] [HUDI-4017] Improve spark sql coverage in CI (#5512) Add GitHub actions tasks to run spark sql UTs under spark 3.1 and 3.2. --- .github/workflows/bot.yml | 8 ++++++++ .../hudi/functional/TestSqlStatement.scala | 4 ++-- .../SpaceCurveOptimizeBenchmark.scala | 4 ++-- ...ase.scala => HoodieSparkSqlTestBase.scala} | 2 +- .../spark/sql/hudi/TestAlterTable.scala | 2 +- .../hudi/TestAlterTableDropPartition.scala | 2 +- .../spark/sql/hudi/TestCompactionTable.scala | 2 +- .../spark/sql/hudi/TestCreateTable.scala | 2 +- .../spark/sql/hudi/TestDeleteTable.scala | 2 +- .../apache/spark/sql/hudi/TestDropTable.scala | 2 +- .../sql/hudi/TestHoodieOptionConfig.scala | 20 +++---------------- .../spark/sql/hudi/TestInsertTable.scala | 2 +- .../sql/hudi/TestMergeIntoLogOnlyTable.scala | 2 +- .../spark/sql/hudi/TestMergeIntoTable.scala | 2 +- .../spark/sql/hudi/TestMergeIntoTable2.scala | 2 +- .../hudi/TestPartialUpdateForMergeInto.scala | 2 +- .../spark/sql/hudi/TestShowPartitions.scala | 2 +- .../apache/spark/sql/hudi/TestSpark3DDL.scala | 2 +- .../apache/spark/sql/hudi/TestSqlConf.scala | 2 +- .../spark/sql/hudi/TestTimeTravelTable.scala | 2 +- .../spark/sql/hudi/TestTruncateTable.scala | 2 +- .../spark/sql/hudi/TestUpdateTable.scala | 2 +- .../procedure/TestCallCommandParser.scala | 4 ++-- .../hudi/procedure/TestCallProcedure.scala | 4 ++-- .../procedure/TestClusteringProcedure.scala | 4 ++-- .../procedure/TestCompactionProcedure.scala | 4 ++-- .../procedure/TestSavepointsProcedure.scala | 4 ++-- 27 files changed, 43 insertions(+), 49 deletions(-) rename hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{TestHoodieSqlBase.scala => HoodieSparkSqlTestBase.scala} (98%) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 29702846b3d2..b76a465d7128 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -59,3 +59,11 @@ jobs: if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 before hadoop upgrade to 3.x run: mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink,hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark + - name: Spark SQL Test + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + FLINK_PROFILE: ${{ matrix.flinkProfile }} + if: ${{ !endsWith(env.SPARK_PROFILE, '2.4') }} # skip test spark 2.4 as it's covered by Azure CI + run: + mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" '-Dtest=org.apache.spark.sql.hudi.Test*' -pl hudi-spark-datasource/hudi-spark diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSqlStatement.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSqlStatement.scala index c451b51ef77c..f8a9cf5fb060 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSqlStatement.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSqlStatement.scala @@ -18,9 +18,9 @@ package org.apache.hudi.functional import org.apache.hudi.common.util.FileIOUtils -import org.apache.spark.sql.hudi.TestHoodieSqlBase +import org.apache.spark.sql.hudi.HoodieSparkSqlTestBase -class TestSqlStatement extends TestHoodieSqlBase { +class TestSqlStatement extends HoodieSparkSqlTestBase { val STATE_INIT = 0 val STATE_SKIP_COMMENT = 1 val STATE_FINISH_COMMENT = 2 diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/SpaceCurveOptimizeBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/SpaceCurveOptimizeBenchmark.scala index d84fad4f2493..273303fdae63 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/SpaceCurveOptimizeBenchmark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/SpaceCurveOptimizeBenchmark.scala @@ -23,7 +23,7 @@ import org.apache.hudi.ColumnStatsIndexHelper.buildColumnStatsTableFor import org.apache.hudi.config.HoodieClusteringConfig.LayoutOptimizationStrategy import org.apache.hudi.sort.SpaceCurveSortingHelper import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.hudi.TestHoodieSqlBase +import org.apache.spark.sql.hudi.HoodieSparkSqlTestBase import org.apache.spark.sql.types.{IntegerType, StructField} import org.junit.jupiter.api.{Disabled, Tag, Test} @@ -31,7 +31,7 @@ import scala.collection.JavaConversions._ import scala.util.Random @Tag("functional") -object SpaceCurveOptimizeBenchmark extends TestHoodieSqlBase { +object SpaceCurveOptimizeBenchmark extends HoodieSparkSqlTestBase { def evalSkippingPercent(tableName: String, co1: String, co2: String, value1: Int, value2: Int): Unit= { val sourceTableDF = spark.sql(s"select * from ${tableName}") diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieSqlBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/HoodieSparkSqlTestBase.scala similarity index 98% rename from hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieSqlBase.scala rename to hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/HoodieSparkSqlTestBase.scala index d1f373db99e5..68fc6d7c41d8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieSqlBase.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/HoodieSparkSqlTestBase.scala @@ -31,7 +31,7 @@ import org.scalatest.{BeforeAndAfterAll, FunSuite, Tag} import java.io.File import java.util.TimeZone -class TestHoodieSqlBase extends FunSuite with BeforeAndAfterAll { +class HoodieSparkSqlTestBase extends FunSuite with BeforeAndAfterAll { org.apache.log4j.Logger.getRootLogger.setLevel(Level.WARN) private lazy val sparkWareHouse = { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTable.scala index 0f2cb547c2fe..6d29ea3f4a13 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTable.scala @@ -22,7 +22,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.{LongType, StructField, StructType} -class TestAlterTable extends TestHoodieSqlBase { +class TestAlterTable extends HoodieSparkSqlTestBase { test("Test Alter Table") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTableDropPartition.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTableDropPartition.scala index ecbbadeeb9a2..677f8632a714 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTableDropPartition.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTableDropPartition.scala @@ -23,7 +23,7 @@ import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.keygen.{ComplexKeyGenerator, SimpleKeyGenerator} import org.apache.spark.sql.SaveMode -class TestAlterTableDropPartition extends TestHoodieSqlBase { +class TestAlterTableDropPartition extends HoodieSparkSqlTestBase { test("Drop non-partitioned table") { val tableName = generateTableName diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCompactionTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCompactionTable.scala index 20238a6e4318..0ef89fc5b9fe 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCompactionTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCompactionTable.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.hudi -class TestCompactionTable extends TestHoodieSqlBase { +class TestCompactionTable extends HoodieSparkSqlTestBase { test("Test compaction table") { withTempDir {tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala index 6b8efb84e32f..e7910fa11585 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.types._ import scala.collection.JavaConverters._ -class TestCreateTable extends TestHoodieSqlBase { +class TestCreateTable extends HoodieSparkSqlTestBase { test("Test Create Managed Hoodie Table") { val databaseName = "hudi_database" diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDeleteTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDeleteTable.scala index b2e888a5f314..4c7c6269667a 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDeleteTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDeleteTable.scala @@ -22,7 +22,7 @@ import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.keygen.SimpleKeyGenerator import org.apache.spark.sql.SaveMode -class TestDeleteTable extends TestHoodieSqlBase { +class TestDeleteTable extends HoodieSparkSqlTestBase { test("Test Delete Table") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDropTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDropTable.scala index c53eb9127c88..ed43d37d0388 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDropTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDropTable.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.hudi -class TestDropTable extends TestHoodieSqlBase { +class TestDropTable extends HoodieSparkSqlTestBase { test("Test Drop Table") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieOptionConfig.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieOptionConfig.scala index 4c0c60385104..14c2245d5be3 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieOptionConfig.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieOptionConfig.scala @@ -19,27 +19,13 @@ package org.apache.spark.sql.hudi import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, OverwriteWithLatestAvroPayload} import org.apache.hudi.common.table.HoodieTableConfig -import org.apache.hudi.testutils.HoodieClientTestBase - -import org.apache.spark.sql.SparkSession +import org.apache.hudi.testutils.SparkClientFunctionalTestHarness import org.apache.spark.sql.types._ - import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.{BeforeEach, Test} - +import org.junit.jupiter.api.Test import org.scalatest.Matchers.intercept -class TestHoodieOptionConfig extends HoodieClientTestBase { - - var spark: SparkSession = _ - - /** - * Setup method running before each test. - */ - @BeforeEach override def setUp() { - initSparkContexts() - spark = sqlContext.sparkSession - } +class TestHoodieOptionConfig extends SparkClientFunctionalTestHarness { @Test def testWithDefaultSqlOptions(): Unit = { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala index 3141208db121..ab75ef563f22 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.SaveMode import java.io.File -class TestInsertTable extends TestHoodieSqlBase { +class TestInsertTable extends HoodieSparkSqlTestBase { test("Test Insert Into") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoLogOnlyTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoLogOnlyTable.scala index 5139825f9428..232b6bbb511c 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoLogOnlyTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoLogOnlyTable.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.hudi import org.apache.hudi.testutils.DataSourceTestUtils -class TestMergeIntoLogOnlyTable extends TestHoodieSqlBase { +class TestMergeIntoLogOnlyTable extends HoodieSparkSqlTestBase { test("Test Query Log Only MOR Table") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable.scala index 28dee88e1f61..992a442f4fda 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.hudi import org.apache.hudi.{DataSourceReadOptions, HoodieDataSourceHelpers} import org.apache.hudi.common.fs.FSUtils -class TestMergeIntoTable extends TestHoodieSqlBase { +class TestMergeIntoTable extends HoodieSparkSqlTestBase { test("Test MergeInto Basic") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala index 5041a543168b..e162368dacc7 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala @@ -21,7 +21,7 @@ import org.apache.hudi.HoodieSparkUtils import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.spark.sql.Row -class TestMergeIntoTable2 extends TestHoodieSqlBase { +class TestMergeIntoTable2 extends HoodieSparkSqlTestBase { test("Test MergeInto for MOR table 2") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestPartialUpdateForMergeInto.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestPartialUpdateForMergeInto.scala index 2524d04ec81f..1af7a162be18 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestPartialUpdateForMergeInto.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestPartialUpdateForMergeInto.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.hudi -class TestPartialUpdateForMergeInto extends TestHoodieSqlBase { +class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase { test("Test Partial Update") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestShowPartitions.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestShowPartitions.scala index 868bfc43d57f..369f3b341adc 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestShowPartitions.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestShowPartitions.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.hudi import org.apache.spark.sql.Row -class TestShowPartitions extends TestHoodieSqlBase { +class TestShowPartitions extends HoodieSparkSqlTestBase { test("Test Show Non Partitioned Table's Partitions") { val tableName = generateTableName diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala index 54163635984b..15fed579bba4 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession} import scala.collection.JavaConversions._ import scala.collection.JavaConverters._ -class TestSpark3DDL extends TestHoodieSqlBase { +class TestSpark3DDL extends HoodieSparkSqlTestBase { def createTestResult(tableName: String): Array[Row] = { spark.sql(s"select * from ${tableName} order by id") diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSqlConf.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSqlConf.scala index 1a8ac0e64589..ac3c49efdd71 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSqlConf.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSqlConf.scala @@ -28,7 +28,7 @@ import java.nio.file.{Files, Paths} import org.scalatest.BeforeAndAfter -class TestSqlConf extends TestHoodieSqlBase with BeforeAndAfter { +class TestSqlConf extends HoodieSparkSqlTestBase with BeforeAndAfter { def setEnv(key: String, value: String): String = { val field = System.getenv().getClass.getDeclaredField("m") diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala index 471ebd6107dc..ce0f17c3f569 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.hudi import org.apache.hudi.HoodieSparkUtils import org.apache.hudi.common.table.HoodieTableMetaClient -class TestTimeTravelTable extends TestHoodieSqlBase { +class TestTimeTravelTable extends HoodieSparkSqlTestBase { test("Test Insert and Update Record with time travel") { if (HoodieSparkUtils.gteqSpark3_2) { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTruncateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTruncateTable.scala index a61d0f822cf4..5dd243079efb 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTruncateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTruncateTable.scala @@ -23,7 +23,7 @@ import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.keygen.{ComplexKeyGenerator, SimpleKeyGenerator} import org.apache.spark.sql.SaveMode -class TestTruncateTable extends TestHoodieSqlBase { +class TestTruncateTable extends HoodieSparkSqlTestBase { test("Test Truncate non-partitioned Table") { Seq("cow", "mor").foreach { tableType => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala index 57c4a972960a..8c709ab37a6e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.hudi -class TestUpdateTable extends TestHoodieSqlBase { +class TestUpdateTable extends HoodieSparkSqlTestBase { test("Test Update Table") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallCommandParser.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallCommandParser.scala index 87814763bf4d..668fb544934d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallCommandParser.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallCommandParser.scala @@ -21,13 +21,13 @@ import com.google.common.collect.ImmutableList import org.apache.hudi.HoodieSparkUtils import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.plans.logical.{CallCommand, NamedArgument, PositionalArgument} -import org.apache.spark.sql.hudi.TestHoodieSqlBase +import org.apache.spark.sql.hudi.HoodieSparkSqlTestBase import org.apache.spark.sql.types.{DataType, DataTypes} import java.math.BigDecimal import scala.collection.JavaConverters -class TestCallCommandParser extends TestHoodieSqlBase { +class TestCallCommandParser extends HoodieSparkSqlTestBase { private val parser = spark.sessionState.sqlParser test("Test Call Produce with Positional Arguments") { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallProcedure.scala index bdf4cbe7ba0f..f75569a1171f 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallProcedure.scala @@ -17,9 +17,9 @@ package org.apache.spark.sql.hudi.procedure -import org.apache.spark.sql.hudi.TestHoodieSqlBase +import org.apache.spark.sql.hudi.HoodieSparkSqlTestBase -class TestCallProcedure extends TestHoodieSqlBase { +class TestCallProcedure extends HoodieSparkSqlTestBase { test("Test Call show_commits Procedure") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala index 621411723346..f975651bd752 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala @@ -24,11 +24,11 @@ import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieTimeli import org.apache.hudi.common.util.{Option => HOption} import org.apache.hudi.{HoodieCLIUtils, HoodieDataSourceHelpers} -import org.apache.spark.sql.hudi.TestHoodieSqlBase +import org.apache.spark.sql.hudi.HoodieSparkSqlTestBase import scala.collection.JavaConverters.asScalaIteratorConverter -class TestClusteringProcedure extends TestHoodieSqlBase { +class TestClusteringProcedure extends HoodieSparkSqlTestBase { test("Test Call run_clustering Procedure By Table") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala index f6e6772d161b..0f6f96f91196 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala @@ -19,9 +19,9 @@ package org.apache.spark.sql.hudi.procedure -import org.apache.spark.sql.hudi.TestHoodieSqlBase +import org.apache.spark.sql.hudi.HoodieSparkSqlTestBase -class TestCompactionProcedure extends TestHoodieSqlBase { +class TestCompactionProcedure extends HoodieSparkSqlTestBase { test("Test Call run_compaction Procedure by Table") { withTempDir { tmp => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestSavepointsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestSavepointsProcedure.scala index 7d60ca018d32..cfc5319c7564 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestSavepointsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestSavepointsProcedure.scala @@ -17,9 +17,9 @@ package org.apache.spark.sql.hudi.procedure -import org.apache.spark.sql.hudi.TestHoodieSqlBase +import org.apache.spark.sql.hudi.HoodieSparkSqlTestBase -class TestSavepointsProcedure extends TestHoodieSqlBase { +class TestSavepointsProcedure extends HoodieSparkSqlTestBase { test("Test Call create_savepoints Procedure") { withTempDir { tmp =>