improve: calculate md5sums from raw (unprocessed) files

mapillary_tools/exif_write.py

@@ -6,6 +6,7 @@
 import logging
 import math
 import typing as T
+from pathlib import Path
 
 import piexif
 
@@ -16,10 +17,14 @@
 class ExifEdit:
     _filename_or_bytes: T.Union[str, bytes]
 
-    def __init__(self, filename_or_bytes: T.Union[str, bytes]) -> None:
+    def __init__(self, filename_or_bytes: T.Union[Path, bytes]) -> None:
         """Initialize the object"""
-        self._filename_or_bytes = filename_or_bytes
-        self._ef: T.Dict = piexif.load(filename_or_bytes)
+        if isinstance(filename_or_bytes, Path):
+            # https://github.com/hMatoba/Piexif/issues/124
+            self._filename_or_bytes = str(filename_or_bytes.resolve())
+        else:
+            self._filename_or_bytes = filename_or_bytes
+        self._ef: T.Dict = piexif.load(self._filename_or_bytes)
 
     @staticmethod
     def decimal_to_dms(
@@ -149,13 +154,12 @@ def dump_image_bytes(self) -> bytes:
         piexif.insert(exif_bytes, self._filename_or_bytes, output)
         return output.read()
 
-    def write(self, filename: T.Optional[str] = None) -> None:
+    def write(self, filename: T.Optional[Path] = None) -> None:
         """Save exif data to file."""
         if filename is None:
-            if isinstance(self._filename_or_bytes, str):
-                filename = self._filename_or_bytes
-            else:
+            if not isinstance(self._filename_or_bytes, str):
                 raise RuntimeError("Unable to write image into bytes")
+            filename = Path(self._filename_or_bytes)
 
         exif_bytes = self._safe_dump()
 
@@ -165,4 +169,4 @@ def write(self, filename: T.Optional[str] = None) -> None:
             with open(self._filename_or_bytes, "rb") as fp:
                 img = fp.read()
 
-        piexif.insert(exif_bytes, img, filename)
+        piexif.insert(exif_bytes, img, str(filename))

mapillary_tools/process_geotag_properties.py

@@ -296,7 +296,7 @@ def _verify_image_exif_write(
     metadata: types.ImageMetadata,
 ) -> types.ImageMetadataOrError:
     with metadata.filename.open("rb") as fp:
-        edit = exif_write.ExifEdit(fp.read())
+        edit = exif_write.ExifEdit(metadata.filename)
     # The cast is to fix the type error in Python3.6:
     # Argument 1 to "add_image_description" of "ExifEdit" has incompatible type "ImageDescription"; expected "Dict[str, Any]"
     edit.add_image_description(
@@ -364,7 +364,7 @@ def _overwrite_exif_tags(
         disable=LOG.getEffectiveLevel() <= logging.DEBUG,
     ):
         try:
-            image_exif = exif_write.ExifEdit(str(metadata.filename))
+            image_exif = exif_write.ExifEdit(metadata.filename)
 
             if all_tags or time_tag:
                 image_exif.add_date_time_original(

mapillary_tools/sample_video.py

@@ -138,6 +138,6 @@ def _sample_single_video(
                 continue
             seconds = frame_idx * sample_interval * duration_ratio
             timestamp = start_time + datetime.timedelta(seconds=seconds)
-            exif_edit = ExifEdit(str(sample_paths[0]))
+            exif_edit = ExifEdit(sample_paths[0])
             exif_edit.add_date_time_original(timestamp)
             exif_edit.write()

mapillary_tools/upload.py

@@ -21,6 +21,7 @@
     geo,
     ipc,
     types,
+    upload_api_v4,
     uploader,
     utils,
 )
@@ -707,6 +708,8 @@ def upload(
             )
             for idx, video_metadata in enumerate(specified_video_metadatas):
                 generator = camm_builder.camm_sample_generator2(video_metadata)
+                with video_metadata.filename.open("rb") as src_fp:
+                    upload_md5sum = utils.md5sum_fp(src_fp).hexdigest()
                 with video_metadata.filename.open("rb") as src_fp:
                     camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator)
                     event_payload: uploader.Progress = {
@@ -716,8 +719,11 @@ def upload(
                         "import_path": str(video_metadata.filename),
                     }
                     try:
-                        cluster_id = mly_uploader.upload_camm_fp(
-                            T.cast(T.BinaryIO, camm_fp), event_payload=event_payload
+                        cluster_id = mly_uploader.upload_stream(
+                            T.cast(T.BinaryIO, camm_fp),
+                            upload_api_v4.ClusterFileType.CAMM,
+                            upload_md5sum,
+                            event_payload=event_payload,
                         )
                     except Exception as ex:
                         raise UploadError(ex) from ex
@@ -803,9 +809,13 @@ def _upload_raw_blackvues_DEPRECATED(
             continue
 
         with video_path.open("rb") as fp:
+            upload_md5sum = utils.md5sum_fp(fp).hexdigest()
             try:
-                cluster_id = mly_uploader.upload_blackvue_fp(
-                    fp, event_payload=event_payload
+                cluster_id = mly_uploader.upload_stream(
+                    fp,
+                    upload_api_v4.ClusterFileType.BLACKVUE,
+                    upload_md5sum,
+                    event_payload=event_payload,
                 )
             except Exception as ex:
                 raise UploadError(ex) from ex
@@ -851,8 +861,12 @@ def _upload_raw_camm_DEPRECATED(
             continue
         try:
             with video_path.open("rb") as fp:
-                cluster_id = mly_uploader.upload_camm_fp(
-                    fp, event_payload=event_payload
+                upload_md5sum = utils.md5sum_fp(fp).hexdigest()
+                cluster_id = mly_uploader.upload_stream(
+                    fp,
+                    upload_api_v4.ClusterFileType.CAMM,
+                    upload_md5sum,
+                    event_payload=event_payload,
                 )
         except Exception as ex:
             raise UploadError(ex) from ex