Hudi fixes and exclusions for databricks (#147)

logicalclocks · Nov 14, 2020 · 70dcc86 · 70dcc86
1 parent a10f6d5
commit 70dcc86
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 5 deletions.
diff --git a/java/pom.xml b/java/pom.xml
@@ -36,6 +36,16 @@
             <groupId>com.logicalclocks</groupId>
             <artifactId>deequ</artifactId>
             <version>${deequ.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>

diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/HudiEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/HudiEngine.java
@@ -195,11 +195,13 @@ private Map<String, String> setupHudiReadOpts(Long startTimestamp, Long endTimes
                                                 Map<String, String> readOptions) {
     Map<String, String> hudiArgs = new HashMap<String, String>();
 
-    String hudiCommitStartTime = timeStampToHudiFormat(startTimestamp);
-    String hudiCommitEndTime = timeStampToHudiFormat(endTimestamp);
+    if (startTimestamp != null) {
+      hudiArgs.put(HUDI_BEGIN_INSTANTTIME_OPT_KEY, timeStampToHudiFormat(startTimestamp));
+    } else {
+      hudiArgs.put(HUDI_BEGIN_INSTANTTIME_OPT_KEY, timeStampToHudiFormat(0L));
+    }
 
-    hudiArgs.put(HUDI_BEGIN_INSTANTTIME_OPT_KEY, hudiCommitStartTime);
-    hudiArgs.put(HUDI_END_INSTANTTIME_OPT_KEY, hudiCommitEndTime);
+    hudiArgs.put(HUDI_END_INSTANTTIME_OPT_KEY, timeStampToHudiFormat(endTimestamp));
     hudiArgs.put(HUDI_QUERY_TYPE_OPT_KEY, HUDI_QUERY_TYPE_INCREMENTAL_OPT_VAL);
 
     // Overwrite with user provided options if any

diff --git a/python/hsfs/core/hudi_feature_group_alias.py b/python/hsfs/core/hudi_feature_group_alias.py
@@ -24,8 +24,8 @@ def __init__(
         self,
         feature_group,
         alias,
-        left_feature_group_start_timestamp,
         left_feature_group_end_timestamp,
+        left_feature_group_start_timestamp=0,
     ):
         self._feature_group = feature_group_module.FeatureGroup.from_response_json(
             feature_group