Restructure video processing and add gpx, nmea and exiftool runtime s…

…upport (#654)

.gitignore

@@ -10,5 +10,6 @@ __pycache__
 /build/
 /dist/
 /venv/
+/.pyre/
 .DS_Store
-*.log
+*.log

README.md

@@ -292,6 +292,92 @@ mapillary_tools video_process MY_VIDEO_DIR \
     --video_sample_distance -1 --video_sample_interval 2
 ```
 
+## New video geotagging features (experimental)
+
+As experimental features, mapillary_tools can now:
+* Geotag videos from tracks recorded in GPX and NMEA files
+* Invoke `exiftool` internally, if available on the system. `exiftool` can extract geolocation data from a wide
+range of video formats, allowing us to support more cameras with less trouble for the end user.
+* Try several geotagging sources sequentially, until proper data is found.
+
+These features apply to the `process` command, which analyzes the video for direct upload instead of sampling it
+into images. They are experimental and will be subject to change in future releases.
+
+### Usage
+
+The new video processing is triggered with the `--video_geotag_source SOURCE` option. It can be specified multiple times:
+In this case, each source will be tried in turn, until one returns good quality data.
+
+`SOURCE` can be:
+1. the plain name of the source - one of `video, camm, gopro, blackvue, gpx, nmea, exiftool_xml, exiftool_runtime`
+2. a JSON object that includes optional parameters
+```json5
+{
+  "source": "SOURCE",
+  "pattern": "FILENAME_PATTERN" // optional
+}
+```
+
+PATTERN specifies how the data source file is named, starting from the video filename:
+* `%f`: the full video filename
+* `%g`: the video filename without extension
+* `%e`: the video filename extension
+
+Supported sources and their default pattern are:
+
+* `video`: parse the video, in order, as `camm, gopro, blackvue`. Pattern: `%f`
+* `camm`: parse the video looking for a CAMM track. Pattern: `%f`
+* `gopro`: parse the video looking for geolocation in GoPro format. Pattern: `%f`
+* `blackvue`: parse the video looking for geolocation in BlackVue format. Pattern: `%f`
+* `gpx`: external GPX file. Pattern: `%g.gpx`
+* `nmea`: external NMEA file. Pattern: `%g.nmea`
+* `exiftool_xml`: external XML file generated by exiftool. Pattern: `%g.xml`
+* `exiftool_runtime`: execute exiftool on the video file. Pattern: `%f`
+
+Notes:
+* `exiftool_runtime` only works if exiftool is installed in the system. If exiftool is not in the default
+execution path, it is possible to specify its location by setting the environment variable `EXIFTOOL_PATH`.
+* pattern are case-sensitive or not depending on the filesystem - in Windows, `%g.gpx` will match both `basename.gpx`
+and `basename.GPX`, in MacOs, Linux or other Unix systems no.
+* If both `--video_geotag_source` and `--geotag_source` are specified, `--video_geotag_source` will apply to video files
+and `--geotag_source` to image files.
+
+### Examples
+
+#### Generic supported videos
+
+Process all videos in a directory, trying to parse them as CAMM, GoPro or BlackVue:
+```sh
+mapillary_tools process --video_geotag_source video VIDEO_DIR/
+```
+
+#### External GPX
+
+Process all videos in a directory, taking geolocation data from GPX files. A video named `foo.mp4` will be associated
+with a GPX file called `foo.gpx`.
+```sh
+mapillary_tools process --video_geotag_source gpx VIDEO_DIR/
+```
+
+#### Insta360 stitched videos
+
+The videos to process have been stitched by Insta360 Studio; the geolocation data is in the original
+videos in the parent directory, and there may be GPX files alongside the stitched video.
+First look for GPX, then fallback to running exiftool against the original videos.
+```sh
+mapillary_tools process \
+--video_geotag_source gpx \
+--video_geotag_source '{"source": "exiftool_runtime", "pattern": "../%g.insv"}' \
+VIDEO_DIR/
+```
+
+### Limitations of `--video_geotag_source`
+
+**External geolocation sources will be aligned with the start of video, there is
+currently no way of applying offsets or scaling the time.** This means, for instance, that GPX tracks must begin precisely
+at the instant of the video, and that timelapse videos are supported only for sources `camm, gopro, blackvue`.
+
+
 ## Authenticate
 
 The command `authenticate` will update the user credentials stored in the config file.

mapillary_tools/commands/__main__.py

@@ -63,14 +63,14 @@ def add_general_arguments(parser, command):
     elif command in ["upload"]:
         parser.add_argument(
             "import_path",
-            help="Path to your images.",
+            help="Paths to your images or videos.",
             nargs="+",
             type=Path,
         )
     elif command in ["process", "process_and_upload"]:
         parser.add_argument(
             "import_path",
-            help="Path to your images.",
+            help="Paths to your images or videos.",
             nargs="+",
             type=Path,
         )

mapillary_tools/commands/process.py

@@ -171,6 +171,13 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser):
             required=False,
             type=Path,
         )
+        group_geotagging.add_argument(
+            "--video_geotag_source",
+            help="Name of the video data extractor and optional arguments. Can be specified multiple times. See the documentation for details. [Experimental, subject to change]",
+            action="append",
+            default=[],
+            required=False,
+        )
         group_geotagging.add_argument(
             "--interpolation_use_gpx_start_time",
             help=f"If supplied, the first image will use the first GPX point time for interpolation, which means the image location will be interpolated to the first GPX point too. Only works for geotagging from {', '.join(geotag_gpx_based_sources)}.",
@@ -261,13 +268,14 @@ def run(self, vars_args: dict):
             vars_args["duplicate_angle"] = 360
 
         metadatas = process_geotag_properties(
+            vars_args=vars_args,
             **(
                 {
                     k: v
                     for k, v in vars_args.items()
                     if k in inspect.getfullargspec(process_geotag_properties).args
                 }
-            )
+            ),
         )
 
         metadatas = process_import_meta_properties(

mapillary_tools/constants.py

@@ -18,6 +18,7 @@
 VIDEO_DURATION_RATIO = float(os.getenv(_ENV_PREFIX + "VIDEO_DURATION_RATIO", 1))
 FFPROBE_PATH: str = os.getenv(_ENV_PREFIX + "FFPROBE_PATH", "ffprobe")
 FFMPEG_PATH: str = os.getenv(_ENV_PREFIX + "FFMPEG_PATH", "ffmpeg")
+EXIFTOOL_PATH: str = os.getenv(_ENV_PREFIX + "EXIFTOOL_PATH", "exiftool")
 IMAGE_DESCRIPTION_FILENAME = os.getenv(
     _ENV_PREFIX + "IMAGE_DESCRIPTION_FILENAME", "mapillary_image_description.json"
 )

mapillary_tools/process_geotag_properties.py

@@ -20,7 +20,10 @@
     geotag_videos_from_exiftool_video,
     geotag_videos_from_video,
 )
-from .types import FileType
+from .types import FileType, VideoMetadataOrError
+
+from .video_data_extraction.cli_options import CliOptions, CliParserOptions
+from .video_data_extraction.extract_video_data import VideoDataExtractor
 
 
 LOG = logging.getLogger(__name__)
@@ -30,6 +33,17 @@
     "gopro_videos", "blackvue_videos", "camm", "exif", "gpx", "nmea", "exiftool"
 ]
 
+VideoGeotagSource = T.Literal[
+    "video",
+    "camm",
+    "gopro",
+    "blackvue",
+    "gpx",
+    "nmea",
+    "exiftool_xml",
+    "exiftool_runtime",
+]
+
 
 def _process_images(
     image_paths: T.Sequence[Path],
@@ -40,7 +54,7 @@ def _process_images(
     interpolation_offset_time: float = 0.0,
     num_processes: T.Optional[int] = None,
     skip_subfolders=False,
-) -> T.List[types.ImageMetadataOrError]:
+) -> T.Sequence[types.ImageMetadataOrError]:
     geotag: geotag_from_generic.GeotagImagesFromGeneric
 
     if video_import_path is not None:
@@ -124,6 +138,37 @@ def _process_images(
     return geotag.to_description()
 
 
+def _process_videos(
+    geotag_source: str,
+    geotag_source_path: T.Optional[Path],
+    video_paths: T.Sequence[Path],
+    num_processes: T.Optional[int],
+    filetypes: T.Optional[T.Set[FileType]],
+) -> T.Sequence[VideoMetadataOrError]:
+    geotag: geotag_from_generic.GeotagVideosFromGeneric
+    if geotag_source == "exiftool":
+        if geotag_source_path is None:
+            raise exceptions.MapillaryFileNotFoundError(
+                "Geotag source path (--geotag_source_path) is required"
+            )
+        if not geotag_source_path.exists():
+            raise exceptions.MapillaryFileNotFoundError(
+                f"Geotag source file not found: {geotag_source_path}"
+            )
+        geotag = geotag_videos_from_exiftool_video.GeotagVideosFromExifToolVideo(
+            video_paths,
+            geotag_source_path,
+            num_processes=num_processes,
+        )
+    else:
+        geotag = geotag_videos_from_video.GeotagVideosFromVideo(
+            video_paths,
+            filetypes=filetypes,
+            num_processes=num_processes,
+        )
+    return geotag.to_description()
+
+
 def _normalize_import_paths(
     import_path: T.Union[Path, T.Sequence[Path]]
 ) -> T.Sequence[Path]:
@@ -137,6 +182,7 @@ def _normalize_import_paths(
 
 
 def process_geotag_properties(
+    vars_args: T.Dict,  # Hello, I'm a hack
     import_path: T.Union[Path, T.Sequence[Path]],
     filetypes: T.Set[FileType],
     geotag_source: GeotagSource,
@@ -170,51 +216,43 @@ def process_geotag_properties(
             skip_subfolders=skip_subfolders,
             check_file_suffix=check_file_suffix,
         )
-        image_metadatas = _process_images(
-            image_paths,
-            geotag_source=geotag_source,
-            geotag_source_path=geotag_source_path,
-            video_import_path=video_import_path,
-            interpolation_use_gpx_start_time=interpolation_use_gpx_start_time,
-            interpolation_offset_time=interpolation_offset_time,
-            num_processes=num_processes,
-            skip_subfolders=skip_subfolders,
-        )
-        metadatas.extend(image_metadatas)
-
-    if (
-        FileType.CAMM in filetypes
-        or FileType.GOPRO in filetypes
-        or FileType.BLACKVUE in filetypes
-        or FileType.VIDEO in filetypes
-    ):
-        video_paths = utils.find_videos(
-            import_paths,
-            skip_subfolders=skip_subfolders,
-            check_file_suffix=check_file_suffix,
-        )
-        geotag: geotag_from_generic.GeotagVideosFromGeneric
-        if geotag_source == "exiftool":
-            if geotag_source_path is None:
-                raise exceptions.MapillaryFileNotFoundError(
-                    "Geotag source path (--geotag_source_path) is required"
-                )
-            if not geotag_source_path.exists():
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Geotag source file not found: {geotag_source_path}"
-                )
-            geotag = geotag_videos_from_exiftool_video.GeotagVideosFromExifToolVideo(
-                video_paths,
-                geotag_source_path,
+        if image_paths:
+            image_metadatas = _process_images(
+                image_paths,
+                geotag_source=geotag_source,
+                geotag_source_path=geotag_source_path,
+                video_import_path=video_import_path,
+                interpolation_use_gpx_start_time=interpolation_use_gpx_start_time,
+                interpolation_offset_time=interpolation_offset_time,
                 num_processes=num_processes,
+                skip_subfolders=skip_subfolders,
             )
-        else:
-            geotag = geotag_videos_from_video.GeotagVideosFromVideo(
-                video_paths,
-                filetypes=filetypes,
-                num_processes=num_processes,
+            metadatas.extend(image_metadatas)
+
+    # --video_geotag_source is still experimental, for videos execute it XOR the legacy code
+    if vars_args["video_geotag_source"]:
+        metadatas.extend(_process_videos_beta(vars_args))
+    else:
+        if (
+            FileType.CAMM in filetypes
+            or FileType.GOPRO in filetypes
+            or FileType.BLACKVUE in filetypes
+            or FileType.VIDEO in filetypes
+        ):
+            video_paths = utils.find_videos(
+                import_paths,
+                skip_subfolders=skip_subfolders,
+                check_file_suffix=check_file_suffix,
             )
-        metadatas.extend(geotag.to_description())
+            if video_paths:
+                video_metadata = _process_videos(
+                    geotag_source,
+                    geotag_source_path,
+                    video_paths,
+                    num_processes,
+                    filetypes,
+                )
+                metadatas.extend(video_metadata)
 
     # filenames should be deduplicated in utils.find_images/utils.find_videos
     assert len(metadatas) == len(
@@ -224,6 +262,38 @@ def process_geotag_properties(
     return metadatas
 
 
+def _process_videos_beta(vars_args: T.Dict):
+    geotag_sources = vars_args["video_geotag_source"]
+    geotag_sources_opts: T.List[CliParserOptions] = []
+    for source in geotag_sources:
+        parsed_opts: CliParserOptions = {}
+        try:
+            parsed_opts = json.loads(source)
+        except ValueError:
+            if source not in T.get_args(VideoGeotagSource):
+                raise exceptions.MapillaryBadParameterError(
+                    "Unknown beta source %s or invalid JSON", source
+                )
+            parsed_opts = {"source": source}
+
+        if "source" not in parsed_opts:
+            raise exceptions.MapillaryBadParameterError("Missing beta source name")
+
+        geotag_sources_opts.append(parsed_opts)
+
+    options: CliOptions = {
+        "paths": vars_args["import_path"],
+        "recursive": vars_args["skip_subfolders"] == False,
+        "geotag_sources_options": geotag_sources_opts,
+        "geotag_source_path": vars_args["geotag_source_path"],
+        "num_processes": vars_args["num_processes"],
+        "device_make": vars_args["device_make"],
+        "device_model": vars_args["device_model"],
+    }
+    extractor = VideoDataExtractor(options)
+    return extractor.process()
+
+
 def _apply_offsets(
     metadatas: T.Iterable[types.ImageMetadata],
     offset_time: float = 0.0,

mapillary_tools/video_data_extraction/cli_options.py

@@ -0,0 +1,22 @@
+import typing as T
+from pathlib import Path
+
+
+known_parser_options = ["source", "pattern", "exiftool_path"]
+
+
+class CliParserOptions(T.TypedDict, total=False):
+    source: str
+    pattern: T.Optional[str]
+    exiftool_path: T.Optional[Path]
+
+
+class CliOptions(T.TypedDict, total=False):
+    paths: T.Sequence[Path]
+    recursive: bool
+    geotag_sources_options: T.Sequence[CliParserOptions]
+    geotag_source_path: Path
+    exiftool_path: Path
+    num_processes: int
+    device_make: T.Optional[str]
+    device_model: T.Optional[str]