Reduce path to working dir

ml6team · Apr 9, 2024 · 6881162 · 6881162
1 parent 22da18c
commit 6881162
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/src/fondant/component/data_io.py b/src/fondant/component/data_io.py
@@ -105,6 +105,12 @@ def load_dataframe(self) -> dd.DataFrame:
 
         for field_name in self.operation_spec.consumes_from_dataset:
             location = self.manifest.get_field_location(field_name)
+
+            if "://" not in location:
+                # If the location is a local path, we need to remove the root path since we only
+                # mount the working dir to the container
+                location = "/" + "/".join(location.split("/")[-4:])
+
             field_mapping[location].append(field_name)
 
         for location, fields in field_mapping.items():