Fix Delta streaming CDC source filter logic to not return incorrect -…

…1 index GitOrigin-RevId: 69028f5f24b2a4f1326475d02219addd1b72363b
delta-io · Oct 14, 2022 · 7420cfc · 7420cfc
1 parent c396a1f
commit 7420cfc
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/core/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceCDCSupport.scala b/core/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceCDCSupport.scala
@@ -45,8 +45,7 @@ trait DeltaSourceCDCSupport { self: DeltaSource =>
     private def moreThanFrom(
         indexedFile: IndexedFile, fromVersion: Long, fromIndex: Long): Boolean = {
       // we need to filter out files so that we get only files after the startingOffset
-      indexedFile.version > fromVersion ||
-        (indexedFile.index == -1 || indexedFile.index > fromIndex)
+      indexedFile.version > fromVersion || indexedFile.index > fromIndex
     }
 
     private def lessThanEnd(