ODP-2324 - Excluding protobuf from spark3

acceldata-io · Sep 26, 2024 · d811013 · d811013
1 parent 8a1f2ee
commit d811013
Show file tree

Hide file tree

Showing 2 changed files with 67 additions and 2 deletions.
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -65,7 +65,7 @@ gcs-connector/hadoop3-2.2.7/shaded/gcs-connector-hadoop3-2.2.7-shaded.jar
 gmetric4j/1.0.10//gmetric4j-1.0.10.jar
 gson/2.11.0//gson-2.11.0.jar
 guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-aliyun/3.3.4//hadoop-aliyun-3.3.4.jar
+
 hadoop-annotations/3.3.4//hadoop-annotations-3.3.4.jar
 hadoop-aws/3.3.4//hadoop-aws-3.3.4.jar
 hadoop-azure-datalake/3.3.4//hadoop-azure-datalake-3.3.4.jar
@@ -95,7 +95,6 @@ hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
 httpclient/4.5.14//httpclient-4.5.14.jar
 httpcore/4.4.16//httpcore-4.4.16.jar
-ini4j/0.5.4//ini4j-0.5.4.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
 ivy/2.5.1//ivy-2.5.1.jar
 jackson-annotations/2.14.2//jackson-annotations-2.14.2.jar

diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
@@ -39,6 +39,28 @@
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <artifactSet>
+                <excludes>
+                  <exclude>org.ini4j:ini4j</exclude> <!-- Exclude ini4j library -->
+                  <exclude>**/protobuf/*.java</exclude>
+                </excludes>
+              </artifactSet>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
   </build>
 
   <dependencies>
@@ -61,11 +83,31 @@
       <artifactId>${hadoop-client-api.artifact}</artifactId>
       <version>${hadoop.version}</version>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ini4j</groupId>
+          <artifactId>ini4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.nimbusds</groupId>
+          <artifactId>nimbus-jose-jwt</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>${hadoop-client-runtime.artifact}</artifactId>
       <version>${hadoop.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.nimbusds</groupId>
+          <artifactId>nimbus-jose-jwt</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.ini4j</groupId>
+          <artifactId>ini4j</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <!--
       the AWS module pulls in jackson; its transitive dependencies can create
@@ -102,6 +144,14 @@
           <groupId>com.amazonaws</groupId>
           <artifactId>aws-java-sdk</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.nimbusds</groupId>
+          <artifactId>nimbus-jose-jwt</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.ini4j</groupId>
+          <artifactId>ini4j</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -110,6 +160,14 @@
       <version>${hadoop.version}</version>
       <scope>${hadoop.deps.scope}</scope>
       <exclusions>
+        <exclusion>
+          <groupId>com.nimbusds</groupId>
+          <artifactId>nimbus-jose-jwt</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.ini4j</groupId>
+          <artifactId>ini4j</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-common</artifactId>
@@ -184,6 +242,10 @@
           <groupId>com.google.guava</groupId>
           <artifactId>guava</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.nimbusds</groupId>
+          <artifactId>nimbus-jose-jwt</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
   </dependencies>
@@ -283,6 +345,10 @@
               <groupId>com.huaweicloud</groupId>
               <artifactId>esdk-obs-java</artifactId>
             </exclusion>
+            <exclusion>
+              <groupId>com.nimbusds</groupId>
+              <artifactId>nimbus-jose-jwt</artifactId>
+            </exclusion>
           </exclusions>
         </dependency>
         <dependency>