[HUDI-3287] Remove hudi-spark dependencies from hudi-kafka-connect-bu…

…ndle (#6079)
apache · Oct 28, 2022 · c6e5fd0 · c6e5fd0
1 parent ae541ba
commit c6e5fd0
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 15 deletions.
diff --git a/...nt-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java b/...nt-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java
@@ -54,7 +54,7 @@ public static KeyGenerator createKeyGenerator(TypedProperties props) throws IOEx
     return Objects.isNull(keyGenerator) ? createAvroKeyGeneratorByType(props) : keyGenerator;
   }
 
-  private static KeyGenerator createAvroKeyGeneratorByType(TypedProperties props) throws IOException {
+  public static KeyGenerator createAvroKeyGeneratorByType(TypedProperties props) throws IOException {
     // Use KeyGeneratorType.SIMPLE as default keyGeneratorType
     String keyGeneratorType =
         props.getString(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), null);

diff --git a/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/utils/KafkaConnectUtils.java b/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/utils/KafkaConnectUtils.java
@@ -18,8 +18,6 @@
 
 package org.apache.hudi.connect.utils;
 
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hudi.client.WriteStatus;
 import org.apache.hudi.common.config.TypedProperties;
 import org.apache.hudi.common.model.HoodieCommitMetadata;
@@ -36,9 +34,11 @@
 import org.apache.hudi.exception.HoodieException;
 import org.apache.hudi.keygen.BaseKeyGenerator;
 import org.apache.hudi.keygen.CustomAvroKeyGenerator;
-import org.apache.hudi.keygen.CustomKeyGenerator;
 import org.apache.hudi.keygen.KeyGenerator;
 import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
+
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.kafka.clients.admin.AdminClient;
 import org.apache.kafka.clients.admin.DescribeTopicsResult;
 import org.apache.kafka.clients.admin.TopicDescription;
@@ -185,8 +185,7 @@ public static String getRecordKeyColumns(KeyGenerator keyGenerator) {
    * @return partition columns Returns the partition columns separated by comma.
    */
   public static String getPartitionColumns(KeyGenerator keyGenerator, TypedProperties typedProperties) {
-
-    if (keyGenerator instanceof CustomKeyGenerator || keyGenerator instanceof CustomAvroKeyGenerator) {
+    if (keyGenerator instanceof CustomAvroKeyGenerator) {
       return ((BaseKeyGenerator) keyGenerator).getPartitionPathFields().stream().map(
           pathField -> Arrays.stream(pathField.split(CustomAvroKeyGenerator.SPLIT_REGEX))
               .findFirst().orElse("Illegal partition path field format: '$pathField' for ${c.getClass.getSimpleName}"))
@@ -200,7 +199,6 @@ public static String getPartitionColumns(KeyGenerator keyGenerator, TypedPropert
     return typedProperties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key());
   }
 
-
   /**
    * Get the Metadata from the latest commit file.
    *

diff --git a/...onnect/src/main/java/org/apache/hudi/connect/writers/KafkaConnectTransactionServices.java b/...onnect/src/main/java/org/apache/hudi/connect/writers/KafkaConnectTransactionServices.java
@@ -84,7 +84,7 @@ public KafkaConnectTransactionServices(KafkaConnectConfigs connectConfigs) throw
     context = new HoodieJavaEngineContext(hadoopConf);
 
     try {
-      KeyGenerator keyGenerator = HoodieAvroKeyGeneratorFactory.createKeyGenerator(
+      KeyGenerator keyGenerator = HoodieAvroKeyGeneratorFactory.createAvroKeyGeneratorByType(
           new TypedProperties(connectConfigs.getProps()));
       String recordKeyFields = KafkaConnectUtils.getRecordKeyColumns(keyGenerator);
       String partitionColumns = KafkaConnectUtils.getPartitionColumns(keyGenerator,

diff --git a/packaging/hudi-kafka-connect-bundle/pom.xml b/packaging/hudi-kafka-connect-bundle/pom.xml
@@ -75,8 +75,6 @@
                                     <include>org.apache.hudi:hudi-common</include>
                                     <include>org.apache.hudi:hudi-client-common</include>
                                     <include>org.apache.hudi:hudi-java-client</include>
-                                    <include>org.apache.hudi:hudi-spark-client</include>
-                                    <include>org.apache.hudi:hudi-spark-common_${scala.binary.version}</include>
                                     <include>org.apache.hudi:hudi-kafka-connect</include>
                                     <include>org.apache.hudi:hudi-utilities_${scala.binary.version}</include>
                                     <include>org.apache.hudi:hudi-hive-sync</include>
@@ -321,11 +319,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hudi</groupId>
-            <artifactId>hudi-spark-common_${scala.binary.version}</artifactId>
-            <version>${project.version}</version>
-        </dependency>
 
         <!-- Avro/ Parquet -->
         <dependency>