ODP-1302 [SPARK-43225][BUILD][SQL] Remove jackson-core-asl and jackso…

…n-mapper-asl from pre-built distribution - Remove `jackson-core-asl` from maven dependency. - Change the scope of `jackson-mapper-asl` from compile to test. - Replace all `Hive.get(conf)` with `Hive.getWithoutRegisterFns(conf)`. To fix CVE issue: https://github.com/apache/spark/security/dependabot/50. No. manual test. Closes apache#40893 from wangyum/SPARK-43225. Lead-authored-by: Yuming Wang <wgyumg@gmail.com> Co-authored-by: Yuming Wang <yumwang@ebay.com> Signed-off-by: Sean Owen <srowen@gmail.com> (cherry picked from commit 9c237d7) [SPARK-43868][SQL][TESTS] Remove `originalUDFs` from `TestHive` to ensure `ObjectHashAggregateExecBenchmark` can run successfully on Github Action This pr remove `originalUDFs` from `TestHive` to ensure `ObjectHashAggregateExecBenchmark` can run successfully on Github Action. After SPARK-43225, `org.codehaus.jackson:jackson-mapper-asl` becomes a test scope dependency, so when using GA to run benchmark, it is not in the classpath because GA uses https://github.com/apache/spark/blob/d61c77cac17029ee27319e6b766b48d314a4dd31/.github/workflows/benchmark.yml#L179-L183 iunstead of the sbt `Test/runMain`. `ObjectHashAggregateExecBenchmark` used `TestHive`, and `TestHive` will always call `org.apache.hadoop.hive.ql.exec.FunctionRegistry#getFunctionNames` to init `originalUDFs` before this pr, so when we run `ObjectHashAggregateExecBenchmark` on GitHub Actions, there will be the following exceptions:
acceldata-io · Apr 9, 2024 · 1c10e28 · 1c10e28
1 parent 5cb61e7
commit 1c10e28
Show file tree

Hide file tree

Showing 10 changed files with 24 additions and 72 deletions.
diff --git a/core/pom.xml b/core/pom.xml
@@ -480,14 +480,6 @@
           <groupId>commons-logging</groupId>
           <artifactId>commons-logging</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-mapper-asl</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-core-asl</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.fasterxml.jackson.core</groupId>
           <artifactId>jackson-core</artifactId>

diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -97,16 +97,14 @@ httpclient/4.5.13//httpclient-4.5.13.jar
 httpcore/4.4.14//httpcore-4.4.14.jar
 ini4j/0.5.4//ini4j-0.5.4.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
-ivy/2.5.0//ivy-2.5.0.jar
-jackson-annotations/2.13.4//jackson-annotations-2.13.4.jar
-jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
-jackson-core/2.13.4//jackson-core-2.13.4.jar
-jackson-databind/2.13.4.1//jackson-databind-2.13.4.1.jar
-jackson-dataformat-cbor/2.13.4//jackson-dataformat-cbor-2.13.4.jar
-jackson-dataformat-yaml/2.13.4//jackson-dataformat-yaml-2.13.4.jar
-jackson-datatype-jsr310/2.13.4//jackson-datatype-jsr310-2.13.4.jar
-jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.12/2.13.4//jackson-module-scala_2.12-2.13.4.jar
+ivy/2.5.1//ivy-2.5.1.jar
+jackson-annotations/2.14.2//jackson-annotations-2.14.2.jar
+jackson-core/2.14.2//jackson-core-2.14.2.jar
+jackson-databind/2.14.2//jackson-databind-2.14.2.jar
+jackson-dataformat-cbor/2.14.2//jackson-dataformat-cbor-2.14.2.jar
+jackson-dataformat-yaml/2.14.2//jackson-dataformat-yaml-2.14.2.jar
+jackson-datatype-jsr310/2.14.2//jackson-datatype-jsr310-2.14.2.jar
+jackson-module-scala_2.12/2.14.2//jackson-module-scala_2.12-2.14.2.jar
 jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
 jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
 jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar

diff --git a/pom.xml b/pom.xml
@@ -179,7 +179,6 @@
     <scalafmt.parameters>--test</scalafmt.parameters>
     <!-- for now, not running scalafmt as part of default verify pipeline -->
     <scalafmt.skip>true</scalafmt.skip>
-    <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
     <fasterxml.jackson.version>2.12.3</fasterxml.jackson.version>
     <snappy.version>1.1.8.4</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
@@ -1154,10 +1153,6 @@
             <groupId>asm</groupId>
             <artifactId>asm</artifactId>
           </exclusion>
-          <exclusion>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-mapper-asl</artifactId>
-          </exclusion>
           <exclusion>
             <groupId>org.ow2.asm</groupId>
             <artifactId>asm</artifactId>
@@ -1602,28 +1597,7 @@
           </exclusion>
         </exclusions>
       </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-core-asl</artifactId>
-        <version>${codehaus.jackson.version}</version>
-        <scope>${hadoop.deps.scope}</scope>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-mapper-asl</artifactId>
-        <version>${codehaus.jackson.version}</version>
-        <scope>${hadoop.deps.scope}</scope>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-xc</artifactId>
-        <version>${codehaus.jackson.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-jaxrs</artifactId>
-        <version>${codehaus.jackson.version}</version>
-      </dependency>
+      <!-- Hive 2.3 need this to init Hive's FunctionRegistry -->
       <dependency>
         <groupId>${hive.group}</groupId>
         <artifactId>hive-beeline</artifactId>

diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
@@ -201,6 +201,16 @@
        <artifactId>jersey-json</artifactId>
        <scope>test</scope>
        <version>${jersey-1.version}</version>
+       <exclusions>
+         <exclusion>
+           <groupId>org.codehaus.jackson</groupId>
+           <artifactId>jackson-core-asl</artifactId>
+         </exclusion>
+         <exclusion>
+           <groupId>org.codehaus.jackson</groupId>
+           <artifactId>jackson-mapper-asl</artifactId>
+         </exclusion>
+       </exclusions>
      </dependency>
      <dependency>
        <groupId>com.sun.jersey</groupId>

diff --git a/sql/core/pom.xml b/sql/core/pom.xml
@@ -255,16 +255,6 @@
     <profile>
       <id>hadoop-2.7</id>
       <dependencies>
-        <dependency>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-core-asl</artifactId>
-          <scope>test</scope>
-        </dependency>
-        <dependency>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-mapper-asl</artifactId>
-          <scope>test</scope>
-        </dependency>
       </dependencies>
     </profile>
   </profiles>

diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/CLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/CLIService.java
@@ -534,7 +534,7 @@ public synchronized String getDelegationTokenFromMetaStore(String owner)
 
     try {
       Hive.closeCurrent();
-      return Hive.get(hiveConf).getDelegationToken(owner, owner);
+      return Hive.getWithoutRegisterFns(hiveConf).getDelegationToken(owner, owner);
     } catch (HiveException e) {
       if (e.getCause() instanceof UnsupportedOperationException) {
         throw (UnsupportedOperationException)e.getCause();

diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -252,7 +252,7 @@ public static int setVariable(String varname, String varvalue) throws Exception
       ss.getHiveVariables().put(propName, substitution.substitute(ss.getConf(),varvalue));
     } else if (varname.startsWith(METACONF_PREFIX)) {
       String propName = varname.substring(METACONF_PREFIX.length());
-      Hive hive = Hive.get(ss.getConf());
+      Hive hive = Hive.getWithoutRegisterFns(ss.getConf());
       hive.setMetaConf(propName, substitution.substitute(ss.getConf(), varvalue));
     } else {
       setConf(varname, varname, varvalue, true);
@@ -413,7 +413,7 @@ public HiveConf getHiveConf() {
   @Override
   public IMetaStoreClient getMetaStoreClient() throws HiveSQLException {
     try {
-      return Hive.get(getHiveConf()).getMSC();
+      return Hive.getWithoutRegisterFns(getHiveConf()).getMSC();
     } catch (HiveException e) {
       throw new HiveSQLException("Failed to get metastore connection", e);
     } catch (MetaException e) {

diff --git a/...hriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/...hriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
@@ -54,7 +54,7 @@ public HiveSessionImplwithUGI(TProtocolVersion protocol, String username, String
     // create a new metastore connection for this particular user session
     Hive.set(null);
     try {
-      sessionHive = Hive.get(getHiveConf());
+      sessionHive = Hive.getWithoutRegisterFns(getHiveConf());
     } catch (HiveException e) {
       throw new HiveSQLException("Failed to setup metastore connection", e);
     }
@@ -140,7 +140,7 @@ private void setDelegationToken(String delegationTokenStr) throws HiveSQLExcepti
   private void cancelDelegationToken() throws HiveSQLException {
     if (delegationTokenStr != null) {
       try {
-        Hive.get(getHiveConf()).cancelDelegationToken(delegationTokenStr);
+        Hive.getWithoutRegisterFns(getHiveConf()).cancelDelegationToken(delegationTokenStr);
       } catch (HiveException e) {
         throw new HiveSQLException("Couldn't cancel delegation token", e);
       }

diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
@@ -143,10 +143,6 @@
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-mapper-asl</artifactId>
-    </dependency>
     <!-- transitive dependencies of hive-exec-core doesn't declare -->
     <dependency>
       <groupId>commons-codec</groupId>

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -19,15 +19,13 @@ package org.apache.spark.sql.hive.test
 
 import java.io.File
 import java.net.URI
-import java.util.{Set => JavaSet}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry
 import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
 
 import org.apache.spark.{SparkConf, SparkContext}
@@ -523,12 +521,6 @@ private[hive] class TestHiveSparkSession(
     }
   }
 
-  /**
-   * Records the UDFs present when the server starts, so we can delete ones that are created by
-   * tests.
-   */
-  protected val originalUDFs: JavaSet[String] = FunctionRegistry.getFunctionNames
-
   /**
    * Resets the test instance by deleting any table, view, temp view, and UDF that have been created
    */