Change subset merging method (#334)

Fixes issues when merging dataframes with different number of partitions
ml6team · Aug 2, 2023 · 7fef744 · 7fef744
1 parent 8cb87fd
commit 7fef744
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/fondant/data_io.py b/src/fondant/data_io.py
@@ -134,7 +134,7 @@ def load_dataframe(self) -> dd.DataFrame:
                 subset_df,
                 left_index=True,
                 right_index=True,
-                how="left",
+                how="inner",
             )
 
         dataframe = self.partition_loaded_dataframe(dataframe)