From 61dc9d991373c01d449a8ed26d9bfd7eb93f9301 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 22 Apr 2024 18:36:49 +0800 Subject: [PATCH] [SPARK-47928][SQL][TEST] Speed up test "Add jar support Ivy URI in SQL" ### What changes were proposed in this pull request? `SQLQuerySuite`/"SPARK-33084: Add jar support Ivy URI in SQL" uses Hive deps to test `ADD JAR` which pulls tons of transitive deps, this PR replaces it with light jars but covers all semantics to speed up the UT. ### Why are the changes needed? Speed up the test, and reduce unnecessary relationships with Hive. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Run UT locally. Before ``` [info] - SPARK-33084: Add jar support Ivy URI in SQL (16 minutes, 55 seconds) ``` After ``` [info] - SPARK-33084: Add jar support Ivy URI in SQL (17 seconds, 783 milliseconds) ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #46150 from pan3793/SPARK-47928. Authored-by: Cheng Pan Signed-off-by: yangjie01 --- .../org/apache/spark/sql/SQLQuerySuite.scala | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index f81369bbad367..78d4b91088a64 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3748,22 +3748,21 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark test("SPARK-33084: Add jar support Ivy URI in SQL") { val sc = spark.sparkContext - val hiveVersion = "2.3.9" // transitive=false, only download specified jar - sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion?transitive=false") - assert(sc.listJars() - .exists(_.contains(s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar"))) + sql(s"ADD JAR ivy://org.springframework:spring-core:6.1.6?transitive=false") + assert(sc.listJars().exists(_.contains("org.springframework_spring-core-6.1.6.jar"))) + assert(!sc.listJars().exists(_.contains("org.springframework_spring-jcl-6.1.6.jar"))) // default transitive=true, test download ivy URL jar return multiple jars - sql("ADD JAR ivy://org.scala-js:scalajs-test-interface_2.12:1.2.0") - assert(sc.listJars().exists(_.contains("scalajs-library_2.12"))) - assert(sc.listJars().exists(_.contains("scalajs-test-interface_2.12"))) - - sql(s"ADD JAR ivy://org.apache.hive:hive-contrib:$hiveVersion" + - "?exclude=org.pentaho:pentaho-aggdesigner-algorithm&transitive=true") - assert(sc.listJars().exists(_.contains(s"org.apache.hive_hive-contrib-$hiveVersion.jar"))) - assert(sc.listJars().exists(_.contains(s"org.apache.hive_hive-exec-$hiveVersion.jar"))) - assert(!sc.listJars().exists(_.contains("org.pentaho.pentaho_aggdesigner-algorithm"))) + sql("ADD JAR ivy://org.awaitility:awaitility:4.2.1") + assert(sc.listJars().exists(_.contains("org.awaitility_awaitility-4.2.1.jar"))) + assert(sc.listJars().exists(_.contains("org.hamcrest_hamcrest-2.1.jar"))) + + sql("ADD JAR ivy://org.junit.jupiter:junit-jupiter:5.10.2" + + "?exclude=org.junit.jupiter:junit-jupiter-engine&transitive=true") + assert(sc.listJars().exists(_.contains("org.junit.jupiter_junit-jupiter-api-5.10.2.jar"))) + assert(sc.listJars().exists(_.contains("org.junit.jupiter_junit-jupiter-params-5.10.2.jar"))) + assert(!sc.listJars().exists(_.contains("org.junit.jupiter_junit-jupiter-engine-5.10.2.jar"))) } test("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") {