From 3026a6562e06dd96db2bb6525824affc4e80d413 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Thu, 17 Aug 2023 13:52:14 +0200 Subject: [PATCH 01/12] fix: Support Adobe coordinates format in XMP tags --- mapillary_tools/exif_read.py | 40 +++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/mapillary_tools/exif_read.py b/mapillary_tools/exif_read.py index 3bc6f3d1..8b5f3b1b 100644 --- a/mapillary_tools/exif_read.py +++ b/mapillary_tools/exif_read.py @@ -1,6 +1,8 @@ import abc import datetime +from fractions import Fraction import logging +import re import typing as T import xml.etree.ElementTree as et from pathlib import Path @@ -47,6 +49,32 @@ def gps_to_decimal(values: T.Tuple[Ratio, Ratio, Ratio]) -> T.Optional[float]: return degrees + minutes / 60 + seconds / 3600 +def parse_coordinate(coord: T.Optional[str]) -> T.Optional[float]: + """ If the coordinate is in decimal degrees, just convert it to float, + otherwise try to parse it from the Adobe format + + """ + + if not coord: + return None + + try: + return float(coord) + except ValueError: + pass + + adobe_format = re.match(r'(\d+),(\d{1,3}\.?\d*)([NSWE])', coord) + if adobe_format: + sign = {'N': 1, 'S': -1, 'E': 1, 'W': -1} + deg = Ratio(int(adobe_format.group(1)), 1) + min_frac = Fraction.from_float(float(adobe_format.group(2))) + min = Ratio(min_frac.numerator, min_frac.denominator) + sec = Ratio(0 ,1) + converted = gps_to_decimal((deg, min, sec)) + if converted: + return converted * sign[adobe_format.group(3)] + + def _parse_iso(dtstr: str) -> T.Optional[datetime.datetime]: try: return datetime.datetime.fromisoformat(dtstr) @@ -378,20 +406,22 @@ def extract_direction(self) -> T.Optional[float]: ) def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]: - lat = self._extract_alternative_fields(["exif:GPSLatitude"], float) + lat = self._extract_alternative_fields(["exif:GPSLatitude"], str) + lat = parse_coordinate(lat) if lat is None: return None - - lon = self._extract_alternative_fields(["exif:GPSLongitude"], float) + + lon = self._extract_alternative_fields(["exif:GPSLongitude"], str) + lon = parse_coordinate(lon) if lon is None: return None ref = self._extract_alternative_fields(["exif:GPSLongitudeRef"], str) - if ref and ref.upper() == "W": + if ref and ref.upper() == "W" and lon > 0: lon = -1 * lon ref = self._extract_alternative_fields(["exif:GPSLatitudeRef"], str) - if ref and ref.upper() == "S": + if ref and ref.upper() == "S" and lat > 0: lat = -1 * lat return lon, lat From c93da3dd3d6d642cbbefcdec78cb34a5500c3046 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Thu, 17 Aug 2023 14:38:53 +0200 Subject: [PATCH 02/12] Add support for coordinates in exiftool namespace XMP-exif; add tests --- mapillary_tools/exiftool_read.py | 6 ++++ tests/data/adobe_coords/adobe_coords.jpg | Bin 0 -> 10863 bytes .../mapillary_image_description.json | 1 + tests/integration/test_process.py | 34 ++++++++++++++++++ tests/integration/test_process_and_upload.py | 2 +- 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tests/data/adobe_coords/adobe_coords.jpg create mode 100644 tests/data/adobe_coords/mapillary_image_description.json diff --git a/mapillary_tools/exiftool_read.py b/mapillary_tools/exiftool_read.py index 2ae67286..bdedb606 100644 --- a/mapillary_tools/exiftool_read.py +++ b/mapillary_tools/exiftool_read.py @@ -310,6 +310,12 @@ def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]: if lon_lat is not None: return lon_lat + lon_lat = self._extract_lon_lat( + "XMP-exif:GPSLongitude", "XMP-exif:GPSLatitude" + ) + if lon_lat is not None: + return lon_lat + return None def _extract_lon_lat( diff --git a/tests/data/adobe_coords/adobe_coords.jpg b/tests/data/adobe_coords/adobe_coords.jpg new file mode 100644 index 0000000000000000000000000000000000000000..67ab54f7815173aa0f19ab15e0215dc5fcb1bcd6 GIT binary patch literal 10863 zcmeHNU5p!76~5k0nlz*|q2WiRg262i+Sr~M+cUP+ZR+3Mt#%V@?IsN%rQ?}v?D4!T&z@T)z0F)x2kbu>`HWSgi4t znjmw6P%Bkye2F|lUU|{C?Qt0K%>WhwXJ7W)Www1bfHlD6#u0J|Ur@#`-|+3Mf$2~H zp97rviqDf_w%89V-6~-UnZNNuv`fVMC-lGA`nz}y$9}8r!gP+_AY!G-p zb1U}G(kfYKv4_R{<%iJU02X-x-j-{E#0h*&lxl*2XuK15oG0Y3%duRji`l_^!}PMx{#%cJ zWZQKowL-yk@``2+XkN9(guc9-+Pi+PuU@~ZR<-Qer?@9~YrEsgn7RtLJ2p(Gw!4z`l-B?=bRo-mEbi5n z>~l!1re#yElrQD^9EQj>F$*2GHmq&6I)Mhftz-jJcXoF2JH@UPh8zHIi^t=SW|Z;i}roP#*@v>NO7tU#$#r!$L+Z7YY;xj zV@`i>LJM2ev8J|4+t;Y+J{pT}(YP0>jm?UKJrY)JC%t7=8B<%y*~(7Np{}c&Bgczf z*s7*Ra+9fTcy80w0yXFu#xY(Y>L5jZinwPCWRw}TtRzINEFvR)Jn@tx^pV2Vu3}md z8#InAYh?IIHrcjZ%eQp`E?XIUdEt55r)^u4q->m@5qZWWG@Q;JrZaO1;$3T5>XdbU zy|t1BCa>vQO(@rSQIdp?)NHovf>5a!o0W2Y%EY zWj;3)26C<>3tFx+sFreaMG<9nSQV>7C6qsJ>ljFik+eZ7TRKmugZ0yUa6L( zcB4`kt96O57qc9qU#lB7RkXeOWMb$PBd~8zX+W8})`sFD4UA@eCCmHRhJlgmpgf2P z)ReAG>CAgiai_Mz21tyCjoYqC9i&{Ktm$Q0*S3bb0r~cLKsAkOeoHgNmOi2mMoGCO zm0^2HXp|f9U%g!^mBbd`C>85fvC(V?qK0DY?q1xE|2|`N(=sgkoTWQ-i)vGK#?+=| zy0*gF3k?kra~jNR)OP#Us;%rfs=~U^Q4A;WUyBZHjGNdm?cIsxOl^k9^MT?J7buSK z9o;ZCn7<+u(1>o<$JW$L;wUwfu3xo*Nb&ue;xuiR_X+!e>jgtQ8TK`_YOt2KsA{2q zu@@uCZ1~ZBUAr=MT$cRx7p6*N@(iV6BvNFC;u)PTw5g1sbb-e6wvu%3jLz=Vp0IcX zZl0kb-syx()G(}_h_-I3wpY#+qff^Z3~R-=E9M} zF&C5+j$~+3$a4XTIC^8taJ-;|%JsNZ)CS5=U(?O0^T8_(V#RYw2qCC*K)J58=DB%!uyy7n9Wnbr( zvV)>U&%y00*`mZ>u8M+mx##Mxx_$Z5S~LGlcQxRo8pdPPAYz_Uu?|EnNc>nPW|+zR zG=|yKPve-+{WPArD#wVh&L32RjDY@2~sq^l8KN;kG+tl z2F+tEEzsc;Qxw2D+ML?gm}O@;{1eM%??1m!5jjYmgIL1kIf%y}5gWQm6??7)M)IST zO730=LV1vSpXm;4^%?KPVxQAb#)Fz($PrZsIZTub`T(YSBZK)Dc>*G;Nw^nSN7TVsaEffCbulHX((1Il|`HvF*9z5jIu0CDU2I*80lg?m~XOgO+pP3xid8k#<$p#kYk|* z>!$ANiV@GSo%0u3bcEG3`Xu|l^-KvEFX(h<4f`fkxri;7&emlUV8oQpvt=1Z%o29H zYdP@lY$Hg$Ja%rrUl_we^^7>H*Gmbn7nsDa_{q!;7FM$Kg{ha_SWi!}RSNa{lQ;U< zvxe@D!q&cAPZ>K?bHuHpnsJ!sxw z*iIVzeGCj{2eW8_DG`%p_bm%EZ*&0aE7-(@tSPP%stC_R`{8-$6TDbsOE{(o&d`T@ zcf`7d3gj=1*&&*DR*R{GwTuE1h<=gMB(X`>2bZGTxZ&OZj= zu)O3jn{cNu>Ly!1BV@i&JKGc`8K=iQi@jLv3zZsQt_h_lc$_Em*~0tKwQ@sfmPA1e zk9DezN~_xD%bkYMk~)$Y+V*~QO{$6F?YgE>8~cSQZWDhqgeCh*R<*oC3#xHL+ZuE6 z)XBM4@aF2n;a$5e>~dNr-2N2I59Ryeu1c*~zMZ=kB>6#IYBZWf3Cn;e17sctYjV3> z?KJ8wHc#JKmwM72W;FUyxm`b^*oLhq1D^;t3ve86Z{4(Tl-zA9-Hq;dNs6&A8Lvite42u$bd3 zIYAEQ4S^5l4Hn}9`?s zYO_vaYk)PkL4(SVC4W2z;5U~Vwxu{J+RbVHv0C=2;Fn6@JBA+m$rmgn)Oqep{Q47e z;wL2rM!!%!#oc@EdElN0?s?#z2kv>`o(KN_JP_Y;p(b|BJGjBJ|L3J2;1J?+ck6s- zeWQ&_T)0bf%TUM8p{EHM<0of!Ao1L#&vQrqf=gQ`$^GPUA}Xpgsdu{@xLC8``S)+U zYd7RqIi_m?(<1Z(8t$^--VO(br`3t=0)8H_wBt?~{B^*mu<^&`s?7HRpRr*e;5Qh4 z;KP4o@W_L`J4UDM{#FxxI|V#9O2C5ztZv(5@Zjbbc}yFd(7*JY4{Hqm5`)La6t;N} zyI6Fbn$Py|{TM!vkX5oyHc6LoWI(1wCkAmz4&Np)4%q^|u|WP|PreJO0i<-ILZU@1 zGA4>APgo4VgD3W_Gc1Z8Ds#Mn`%Zt{-~Z>QaX0L%guH!YfB!$<-rs-wRovP71MbC{ zvAlJQkjlS+zY*ge`xfryeVLFSy&mHRUnAtlFA?&wH`$)eYX>*2$kK2~$UnYI$OD&f z*Y1~u9Q%UrH_#ZJeCM}>^x>@3K|DH6-{(F6( zB=cT%?EjpcJPPBv<;+R4bTYGiGPD0W`LH*Jm>;5N3FuR7WX(SNzaz(&Gi2$+@(CpG z4UhOq#(H9&^=Yskg@VlCr6Y{tWIDrgCUZ7(5W`Z$u&{~y&iYL(WXPQT)RE9~FWxYo TJ-4yz#O>_z84hR`3~&AynK?^8 literal 0 HcmV?d00001 diff --git a/tests/data/adobe_coords/mapillary_image_description.json b/tests/data/adobe_coords/mapillary_image_description.json new file mode 100644 index 00000000..0cf2b18b --- /dev/null +++ b/tests/data/adobe_coords/mapillary_image_description.json @@ -0,0 +1 @@ +[{"filename": "/home/adalvit/workspace/mapillary_tools/tests/data/adobe_coords/adobe_coords.jpg", "md5sum": "2be0d38d9eb7c0ac8bca848361787843", "filetype": "image", "MAPLatitude": -0.0702668, "MAPLongitude": 34.3819352, "MAPCaptureTime": "2019_07_16_08_26_11_000", "MAPCompassHeading": {"TrueHeading": 0.0, "MagneticHeading": 0.0}, "MAPSequenceUUID": "0", "MAPDeviceMake": "SAMSUNG", "MAPDeviceModel": "SM-C200", "MAPOrientation": 1}] \ No newline at end of file diff --git a/tests/integration/test_process.py b/tests/integration/test_process.py index c6e0c73a..187119d1 100644 --- a/tests/integration/test_process.py +++ b/tests/integration/test_process.py @@ -59,6 +59,19 @@ "MAPDeviceModel": "VIRB 360", "MAPOrientation": 1, }, + "adobe_coords.jpg": { + "filetype": "image", + "MAPLatitude": -0.0702668, + "MAPLongitude": 34.3819352, + "MAPCaptureTime": "2019_07_16_08_26_11_000", + "MAPCompassHeading": { + "TrueHeading": 0, + "MagneticHeading": 0 + }, + "MAPDeviceMake": "SAMSUNG", + "MAPDeviceModel": "SM-C200", + "MAPOrientation": 1 + } } @@ -260,6 +273,27 @@ def test_angle_with_offset_with_exiftool(setup_data: py.path.local): return test_angle_with_offset(setup_data, use_exiftool=True) +def test_parse_adobe_coordinates(setup_data: py.path.local): + args = f"{EXECUTABLE} process --file_types=image {PROCESS_FLAGS} {setup_data}/adobe_coords" + x = subprocess.run(args, shell=True) + verify_descs([{ + "filename": str(Path(setup_data, "adobe_coords", "adobe_coords.jpg")), + "filetype": "image", + "MAPLatitude": -0.0702668, + "MAPLongitude": 34.3819352, + "MAPCaptureTime": "2019_07_16_08_26_11_000", + "MAPCompassHeading": { + "TrueHeading": 0.0, + "MagneticHeading": 0.0 + }, + "MAPDeviceMake": "SAMSUNG", + "MAPDeviceModel": "SM-C200", + "MAPOrientation": 1, + }], + Path(setup_data, "adobe_coords/mapillary_image_description.json"), + ) + + def test_zip(tmpdir: py.path.local, setup_data: py.path.local): zip_dir = tmpdir.mkdir("zip_dir") x = subprocess.run( diff --git a/tests/integration/test_process_and_upload.py b/tests/integration/test_process_and_upload.py index 5487f7d2..d97b7718 100644 --- a/tests/integration/test_process_and_upload.py +++ b/tests/integration/test_process_and_upload.py @@ -181,7 +181,7 @@ def test_process_and_upload_images_only( setup_upload: py.path.local, ): x = subprocess.run( - f"{EXECUTABLE} --verbose process_and_upload --filetypes=image {UPLOAD_FLAGS} {PROCESS_FLAGS} {setup_data} {setup_data} {setup_data}/images/DSC00001.JPG --desc_path=-", + f"{EXECUTABLE} --verbose process_and_upload --filetypes=image {UPLOAD_FLAGS} {PROCESS_FLAGS} {setup_data}/images {setup_data}/images {setup_data}/images/DSC00001.JPG --desc_path=-", shell=True, ) assert x.returncode == 0, x.stderr From 127643561fbf4632810540317436d55b885f1481 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Thu, 17 Aug 2023 15:28:25 +0200 Subject: [PATCH 03/12] linting --- mapillary_tools/exif_read.py | 20 ++++++++-------- mapillary_tools/exiftool_read.py | 4 +--- tests/integration/test_process.py | 39 ++++++++++++++----------------- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/mapillary_tools/exif_read.py b/mapillary_tools/exif_read.py index 8b5f3b1b..8a738334 100644 --- a/mapillary_tools/exif_read.py +++ b/mapillary_tools/exif_read.py @@ -1,10 +1,10 @@ import abc import datetime -from fractions import Fraction import logging import re import typing as T import xml.etree.ElementTree as et +from fractions import Fraction from pathlib import Path import exifread @@ -50,26 +50,26 @@ def gps_to_decimal(values: T.Tuple[Ratio, Ratio, Ratio]) -> T.Optional[float]: def parse_coordinate(coord: T.Optional[str]) -> T.Optional[float]: - """ If the coordinate is in decimal degrees, just convert it to float, - otherwise try to parse it from the Adobe format - + """If the coordinate is in decimal degrees, just convert it to float, + otherwise try to parse it from the Adobe format + """ if not coord: return None - + try: return float(coord) except ValueError: pass - - adobe_format = re.match(r'(\d+),(\d{1,3}\.?\d*)([NSWE])', coord) + + adobe_format = re.match(r"(\d+),(\d{1,3}\.?\d*)([NSWE])", coord) if adobe_format: - sign = {'N': 1, 'S': -1, 'E': 1, 'W': -1} + sign = {"N": 1, "S": -1, "E": 1, "W": -1} deg = Ratio(int(adobe_format.group(1)), 1) min_frac = Fraction.from_float(float(adobe_format.group(2))) min = Ratio(min_frac.numerator, min_frac.denominator) - sec = Ratio(0 ,1) + sec = Ratio(0, 1) converted = gps_to_decimal((deg, min, sec)) if converted: return converted * sign[adobe_format.group(3)] @@ -410,7 +410,7 @@ def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]: lat = parse_coordinate(lat) if lat is None: return None - + lon = self._extract_alternative_fields(["exif:GPSLongitude"], str) lon = parse_coordinate(lon) if lon is None: diff --git a/mapillary_tools/exiftool_read.py b/mapillary_tools/exiftool_read.py index bdedb606..5906d8ae 100644 --- a/mapillary_tools/exiftool_read.py +++ b/mapillary_tools/exiftool_read.py @@ -310,9 +310,7 @@ def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]: if lon_lat is not None: return lon_lat - lon_lat = self._extract_lon_lat( - "XMP-exif:GPSLongitude", "XMP-exif:GPSLatitude" - ) + lon_lat = self._extract_lon_lat("XMP-exif:GPSLongitude", "XMP-exif:GPSLatitude") if lon_lat is not None: return lon_lat diff --git a/tests/integration/test_process.py b/tests/integration/test_process.py index 187119d1..de21296d 100644 --- a/tests/integration/test_process.py +++ b/tests/integration/test_process.py @@ -64,14 +64,11 @@ "MAPLatitude": -0.0702668, "MAPLongitude": 34.3819352, "MAPCaptureTime": "2019_07_16_08_26_11_000", - "MAPCompassHeading": { - "TrueHeading": 0, - "MagneticHeading": 0 - }, + "MAPCompassHeading": {"TrueHeading": 0, "MagneticHeading": 0}, "MAPDeviceMake": "SAMSUNG", "MAPDeviceModel": "SM-C200", - "MAPOrientation": 1 - } + "MAPOrientation": 1, + }, } @@ -276,21 +273,21 @@ def test_angle_with_offset_with_exiftool(setup_data: py.path.local): def test_parse_adobe_coordinates(setup_data: py.path.local): args = f"{EXECUTABLE} process --file_types=image {PROCESS_FLAGS} {setup_data}/adobe_coords" x = subprocess.run(args, shell=True) - verify_descs([{ - "filename": str(Path(setup_data, "adobe_coords", "adobe_coords.jpg")), - "filetype": "image", - "MAPLatitude": -0.0702668, - "MAPLongitude": 34.3819352, - "MAPCaptureTime": "2019_07_16_08_26_11_000", - "MAPCompassHeading": { - "TrueHeading": 0.0, - "MagneticHeading": 0.0 - }, - "MAPDeviceMake": "SAMSUNG", - "MAPDeviceModel": "SM-C200", - "MAPOrientation": 1, - }], - Path(setup_data, "adobe_coords/mapillary_image_description.json"), + verify_descs( + [ + { + "filename": str(Path(setup_data, "adobe_coords", "adobe_coords.jpg")), + "filetype": "image", + "MAPLatitude": -0.0702668, + "MAPLongitude": 34.3819352, + "MAPCaptureTime": "2019_07_16_08_26_11_000", + "MAPCompassHeading": {"TrueHeading": 0.0, "MagneticHeading": 0.0}, + "MAPDeviceMake": "SAMSUNG", + "MAPDeviceModel": "SM-C200", + "MAPOrientation": 1, + } + ], + Path(setup_data, "adobe_coords/mapillary_image_description.json"), ) From daa061f0b56b1264857bd7d80fa0fce09da6f2f7 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Thu, 17 Aug 2023 15:37:27 +0200 Subject: [PATCH 04/12] Fix typehints --- mapillary_tools/exif_read.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mapillary_tools/exif_read.py b/mapillary_tools/exif_read.py index 8a738334..fd1a86fc 100644 --- a/mapillary_tools/exif_read.py +++ b/mapillary_tools/exif_read.py @@ -74,6 +74,7 @@ def parse_coordinate(coord: T.Optional[str]) -> T.Optional[float]: if converted: return converted * sign[adobe_format.group(3)] + return None def _parse_iso(dtstr: str) -> T.Optional[datetime.datetime]: try: @@ -406,13 +407,13 @@ def extract_direction(self) -> T.Optional[float]: ) def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]: - lat = self._extract_alternative_fields(["exif:GPSLatitude"], str) - lat = parse_coordinate(lat) + lat_str: T.Optional[str] = self._extract_alternative_fields(["exif:GPSLatitude"], str) + lat: T.Optional[float] = parse_coordinate(lat_str) if lat is None: return None - lon = self._extract_alternative_fields(["exif:GPSLongitude"], str) - lon = parse_coordinate(lon) + lon_str: T.Optional[str] = self._extract_alternative_fields(["exif:GPSLongitude"], str) + lon = parse_coordinate(lon_str) if lon is None: return None From 8a01d8179abba0f80b9e3ce123c9d1572dfbcd23 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Thu, 17 Aug 2023 15:44:11 +0200 Subject: [PATCH 05/12] Linting --- mapillary_tools/exif_read.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mapillary_tools/exif_read.py b/mapillary_tools/exif_read.py index fd1a86fc..afc6e96d 100644 --- a/mapillary_tools/exif_read.py +++ b/mapillary_tools/exif_read.py @@ -76,6 +76,7 @@ def parse_coordinate(coord: T.Optional[str]) -> T.Optional[float]: return None + def _parse_iso(dtstr: str) -> T.Optional[datetime.datetime]: try: return datetime.datetime.fromisoformat(dtstr) @@ -407,12 +408,16 @@ def extract_direction(self) -> T.Optional[float]: ) def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]: - lat_str: T.Optional[str] = self._extract_alternative_fields(["exif:GPSLatitude"], str) + lat_str: T.Optional[str] = self._extract_alternative_fields( + ["exif:GPSLatitude"], str + ) lat: T.Optional[float] = parse_coordinate(lat_str) if lat is None: return None - lon_str: T.Optional[str] = self._extract_alternative_fields(["exif:GPSLongitude"], str) + lon_str: T.Optional[str] = self._extract_alternative_fields( + ["exif:GPSLongitude"], str + ) lon = parse_coordinate(lon_str) if lon is None: return None From a856eb426976d78a2f818142feaf2644d55c7356 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Thu, 17 Aug 2023 15:53:18 +0200 Subject: [PATCH 06/12] Fix test data --- tests/data/adobe_coords/mapillary_image_description.json | 1 - tests/integration/test_process.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 tests/data/adobe_coords/mapillary_image_description.json diff --git a/tests/data/adobe_coords/mapillary_image_description.json b/tests/data/adobe_coords/mapillary_image_description.json deleted file mode 100644 index 0cf2b18b..00000000 --- a/tests/data/adobe_coords/mapillary_image_description.json +++ /dev/null @@ -1 +0,0 @@ -[{"filename": "/home/adalvit/workspace/mapillary_tools/tests/data/adobe_coords/adobe_coords.jpg", "md5sum": "2be0d38d9eb7c0ac8bca848361787843", "filetype": "image", "MAPLatitude": -0.0702668, "MAPLongitude": 34.3819352, "MAPCaptureTime": "2019_07_16_08_26_11_000", "MAPCompassHeading": {"TrueHeading": 0.0, "MagneticHeading": 0.0}, "MAPSequenceUUID": "0", "MAPDeviceMake": "SAMSUNG", "MAPDeviceModel": "SM-C200", "MAPOrientation": 1}] \ No newline at end of file diff --git a/tests/integration/test_process.py b/tests/integration/test_process.py index de21296d..136e4739 100644 --- a/tests/integration/test_process.py +++ b/tests/integration/test_process.py @@ -63,7 +63,7 @@ "filetype": "image", "MAPLatitude": -0.0702668, "MAPLongitude": 34.3819352, - "MAPCaptureTime": "2019_07_16_08_26_11_000", + "MAPCaptureTime": "2019_07_16_10_26_11_000", "MAPCompassHeading": {"TrueHeading": 0, "MagneticHeading": 0}, "MAPDeviceMake": "SAMSUNG", "MAPDeviceModel": "SM-C200", From fc2736614b76c8fab5c10ae0733d9c444ad33419 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Thu, 17 Aug 2023 16:16:16 +0200 Subject: [PATCH 07/12] Fix test data again --- tests/integration/test_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_process.py b/tests/integration/test_process.py index 136e4739..7564f8dc 100644 --- a/tests/integration/test_process.py +++ b/tests/integration/test_process.py @@ -280,7 +280,7 @@ def test_parse_adobe_coordinates(setup_data: py.path.local): "filetype": "image", "MAPLatitude": -0.0702668, "MAPLongitude": 34.3819352, - "MAPCaptureTime": "2019_07_16_08_26_11_000", + "MAPCaptureTime": "2019_07_16_10_26_11_000", "MAPCompassHeading": {"TrueHeading": 0.0, "MagneticHeading": 0.0}, "MAPDeviceMake": "SAMSUNG", "MAPDeviceModel": "SM-C200", From ce36f4c34331431fe077f8285d54e75fa5d5a3b8 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Thu, 17 Aug 2023 16:17:15 +0200 Subject: [PATCH 08/12] Apply tz fix to test --- tests/integration/test_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_process.py b/tests/integration/test_process.py index 7564f8dc..564bdb0f 100644 --- a/tests/integration/test_process.py +++ b/tests/integration/test_process.py @@ -280,7 +280,7 @@ def test_parse_adobe_coordinates(setup_data: py.path.local): "filetype": "image", "MAPLatitude": -0.0702668, "MAPLongitude": 34.3819352, - "MAPCaptureTime": "2019_07_16_10_26_11_000", + "MAPCaptureTime": _local_to_utc("2019_07_16_10_26_11_000"), "MAPCompassHeading": {"TrueHeading": 0.0, "MagneticHeading": 0.0}, "MAPDeviceMake": "SAMSUNG", "MAPDeviceModel": "SM-C200", From 12c8c1f2bb3c2a33cf47fd7d4200c3b1b2383445 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Thu, 17 Aug 2023 16:18:32 +0200 Subject: [PATCH 09/12] Apply tz fix to test --- tests/integration/test_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_process.py b/tests/integration/test_process.py index 564bdb0f..ff0be6fc 100644 --- a/tests/integration/test_process.py +++ b/tests/integration/test_process.py @@ -280,7 +280,7 @@ def test_parse_adobe_coordinates(setup_data: py.path.local): "filetype": "image", "MAPLatitude": -0.0702668, "MAPLongitude": 34.3819352, - "MAPCaptureTime": _local_to_utc("2019_07_16_10_26_11_000"), + "MAPCaptureTime": _local_to_utc("2019-07-16T10:26:11"), "MAPCompassHeading": {"TrueHeading": 0.0, "MagneticHeading": 0.0}, "MAPDeviceMake": "SAMSUNG", "MAPDeviceModel": "SM-C200", From 5baba43499f51360cb155c07109a2cc92073584a Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Fri, 18 Aug 2023 10:42:57 +0200 Subject: [PATCH 10/12] PR: compile regex, split function --- mapillary_tools/exif_read.py | 43 +++++++++++++++++++++--------------- tests/unit/test_exifread.py | 25 ++++++++++++++++++++- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/mapillary_tools/exif_read.py b/mapillary_tools/exif_read.py index afc6e96d..1f7bb1a7 100644 --- a/mapillary_tools/exif_read.py +++ b/mapillary_tools/exif_read.py @@ -24,6 +24,8 @@ EXIFREAD_LOG = logging.getLogger("exifread") EXIFREAD_LOG.setLevel(logging.ERROR) +adobe_format_regex = re.compile(r"(\d+),(\d{1,3}\.?\d*)([NSWE])") + def eval_frac(value: Ratio) -> float: return float(value.num) / float(value.den) @@ -49,34 +51,39 @@ def gps_to_decimal(values: T.Tuple[Ratio, Ratio, Ratio]) -> T.Optional[float]: return degrees + minutes / 60 + seconds / 3600 -def parse_coordinate(coord: T.Optional[str]) -> T.Optional[float]: - """If the coordinate is in decimal degrees, just convert it to float, - otherwise try to parse it from the Adobe format - - """ - - if not coord: - return None - +def _parse_coord_numeric(coord: str) -> T.Optional[float]: try: return float(coord) except ValueError: pass - adobe_format = re.match(r"(\d+),(\d{1,3}\.?\d*)([NSWE])", coord) - if adobe_format: + +def _parse_coord_adobe(coord: str) -> T.Optional[float]: + """ + Parse Adobe coordinate format: + """ + matches = adobe_format_regex.match(coord) + if matches: sign = {"N": 1, "S": -1, "E": 1, "W": -1} - deg = Ratio(int(adobe_format.group(1)), 1) - min_frac = Fraction.from_float(float(adobe_format.group(2))) + deg = Ratio(int(matches.group(1)), 1) + min_frac = Fraction.from_float(float(matches.group(2))) min = Ratio(min_frac.numerator, min_frac.denominator) sec = Ratio(0, 1) converted = gps_to_decimal((deg, min, sec)) - if converted: - return converted * sign[adobe_format.group(3)] - + if converted is not None: + return converted * sign[matches.group(3)] return None +def _parse_coord(coord: T.Optional[str]) -> T.Optional[float]: + if coord is None: + return None + parsed = _parse_coord_numeric(coord) + if parsed is None: + parsed = _parse_coord_adobe(coord) + return parsed + + def _parse_iso(dtstr: str) -> T.Optional[datetime.datetime]: try: return datetime.datetime.fromisoformat(dtstr) @@ -411,14 +418,14 @@ def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]: lat_str: T.Optional[str] = self._extract_alternative_fields( ["exif:GPSLatitude"], str ) - lat: T.Optional[float] = parse_coordinate(lat_str) + lat: T.Optional[float] = _parse_coord(lat_str) if lat is None: return None lon_str: T.Optional[str] = self._extract_alternative_fields( ["exif:GPSLongitude"], str ) - lon = parse_coordinate(lon_str) + lon = _parse_coord(lon_str) if lon is None: return None diff --git a/tests/unit/test_exifread.py b/tests/unit/test_exifread.py index 3855bf4f..947e4ca0 100644 --- a/tests/unit/test_exifread.py +++ b/tests/unit/test_exifread.py @@ -8,10 +8,16 @@ import pytest from mapillary_tools import geo -from mapillary_tools.exif_read import ExifRead, parse_datetimestr_with_subsec_and_offset +from mapillary_tools.exif_read import ( + ExifRead, + _parse_coord, + parse_datetimestr_with_subsec_and_offset, +) from mapillary_tools.exif_write import ExifEdit from PIL import ExifTags, Image +import typing as T + """Initialize all the neccessary data""" this_file = os.path.abspath(__file__) @@ -250,6 +256,23 @@ def test_parse(): assert str(dt) == "2021-10-10 17:29:54.124000-02:00", dt +@pytest.mark.parametrize( + "raw_coord,expected", + [ + ("0.0", 0), + ("foo", None), + ("1.5", 1.5), + ("-1.5", -1.5), + ("33,18.32N", 33.30533), + ("33,18.32S", -33.30533), + ("44,24.54E", 44.40900), + ("44,24.54W", -44.40900), + ], +) +def test_parse_coordinates(raw_coord: T.Optional[str], expected: T.Optional[float]): + assert _parse_coord(raw_coord) == pytest.approx(expected) + + # test ExifWrite write a timestamp and ExifRead read it back def test_read_and_write(setup_data: py.path.local): image_path = Path(setup_data, "test_exif.jpg") From 860cc6d55eff296e0dc67a9bf066bab6f92b1be3 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Fri, 18 Aug 2023 11:06:26 +0200 Subject: [PATCH 11/12] fix --- mapillary_tools/exif_read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapillary_tools/exif_read.py b/mapillary_tools/exif_read.py index 1f7bb1a7..d7efc164 100644 --- a/mapillary_tools/exif_read.py +++ b/mapillary_tools/exif_read.py @@ -55,7 +55,7 @@ def _parse_coord_numeric(coord: str) -> T.Optional[float]: try: return float(coord) except ValueError: - pass + return None def _parse_coord_adobe(coord: str) -> T.Optional[float]: From e7571bfc5026aba88eb4488b2564d9ed56c71702 Mon Sep 17 00:00:00 2001 From: Alessandro Dalvit Date: Tue, 22 Aug 2023 11:07:00 +0200 Subject: [PATCH 12/12] Minor refactoring --- mapillary_tools/exif_read.py | 33 +++++++++++++-------------------- tests/unit/test_exifread.py | 36 ++++++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/mapillary_tools/exif_read.py b/mapillary_tools/exif_read.py index d7efc164..69644cb3 100644 --- a/mapillary_tools/exif_read.py +++ b/mapillary_tools/exif_read.py @@ -23,8 +23,8 @@ # https://github.com/ianare/exif-py/issues/167 EXIFREAD_LOG = logging.getLogger("exifread") EXIFREAD_LOG.setLevel(logging.ERROR) - -adobe_format_regex = re.compile(r"(\d+),(\d{1,3}\.?\d*)([NSWE])") +SIGN_BY_DIRECTION = {None: 1, "N": 1, "S": -1, "E": 1, "W": -1} +ADOBE_FORMAT_REGEX = re.compile(r"(\d+),(\d{1,3}\.?\d*)([NSWE])") def eval_frac(value: Ratio) -> float: @@ -51,10 +51,10 @@ def gps_to_decimal(values: T.Tuple[Ratio, Ratio, Ratio]) -> T.Optional[float]: return degrees + minutes / 60 + seconds / 3600 -def _parse_coord_numeric(coord: str) -> T.Optional[float]: +def _parse_coord_numeric(coord: str, ref: T.Optional[str]) -> T.Optional[float]: try: - return float(coord) - except ValueError: + return float(coord) * SIGN_BY_DIRECTION[ref] + except (ValueError, KeyError): return None @@ -62,23 +62,22 @@ def _parse_coord_adobe(coord: str) -> T.Optional[float]: """ Parse Adobe coordinate format: """ - matches = adobe_format_regex.match(coord) + matches = ADOBE_FORMAT_REGEX.match(coord) if matches: - sign = {"N": 1, "S": -1, "E": 1, "W": -1} deg = Ratio(int(matches.group(1)), 1) min_frac = Fraction.from_float(float(matches.group(2))) min = Ratio(min_frac.numerator, min_frac.denominator) sec = Ratio(0, 1) converted = gps_to_decimal((deg, min, sec)) if converted is not None: - return converted * sign[matches.group(3)] + return converted * SIGN_BY_DIRECTION[matches.group(3)] return None -def _parse_coord(coord: T.Optional[str]) -> T.Optional[float]: +def _parse_coord(coord: T.Optional[str], ref: T.Optional[str]) -> T.Optional[float]: if coord is None: return None - parsed = _parse_coord_numeric(coord) + parsed = _parse_coord_numeric(coord, ref) if parsed is None: parsed = _parse_coord_adobe(coord) return parsed @@ -415,28 +414,22 @@ def extract_direction(self) -> T.Optional[float]: ) def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]: + lat_ref = self._extract_alternative_fields(["exif:GPSLatitudeRef"], str) lat_str: T.Optional[str] = self._extract_alternative_fields( ["exif:GPSLatitude"], str ) - lat: T.Optional[float] = _parse_coord(lat_str) + lat: T.Optional[float] = _parse_coord(lat_str, lat_ref) if lat is None: return None + lon_ref = self._extract_alternative_fields(["exif:GPSLongitudeRef"], str) lon_str: T.Optional[str] = self._extract_alternative_fields( ["exif:GPSLongitude"], str ) - lon = _parse_coord(lon_str) + lon = _parse_coord(lon_str, lon_ref) if lon is None: return None - ref = self._extract_alternative_fields(["exif:GPSLongitudeRef"], str) - if ref and ref.upper() == "W" and lon > 0: - lon = -1 * lon - - ref = self._extract_alternative_fields(["exif:GPSLatitudeRef"], str) - if ref and ref.upper() == "S" and lat > 0: - lat = -1 * lat - return lon, lat def extract_make(self) -> T.Optional[str]: diff --git a/tests/unit/test_exifread.py b/tests/unit/test_exifread.py index 947e4ca0..62e6a19e 100644 --- a/tests/unit/test_exifread.py +++ b/tests/unit/test_exifread.py @@ -1,5 +1,7 @@ import datetime import os + +import typing as T import unittest from pathlib import Path @@ -9,15 +11,13 @@ from mapillary_tools import geo from mapillary_tools.exif_read import ( - ExifRead, _parse_coord, + ExifRead, parse_datetimestr_with_subsec_and_offset, ) from mapillary_tools.exif_write import ExifEdit from PIL import ExifTags, Image -import typing as T - """Initialize all the neccessary data""" this_file = os.path.abspath(__file__) @@ -257,20 +257,28 @@ def test_parse(): @pytest.mark.parametrize( - "raw_coord,expected", + "raw_coord,raw_ref,expected", [ - ("0.0", 0), - ("foo", None), - ("1.5", 1.5), - ("-1.5", -1.5), - ("33,18.32N", 33.30533), - ("33,18.32S", -33.30533), - ("44,24.54E", 44.40900), - ("44,24.54W", -44.40900), + (None, "", None), + ("foo", "N", None), + ("0.0", "foo", None), + ("0.0", "N", 0), + ("1.5", "N", 1.5), + ("1.5", "S", -1.5), + ("-1.5", "N", -1.5), + ("-1.5", "S", 1.5), + ("-1.5", "S", 1.5), + ("33,18.32N", "N", 33.30533), + ("33,18.32N", "S", 33.30533), + ("33,18.32S", "", -33.30533), + ("44,24.54E", "", 44.40900), + ("44,24.54W", "", -44.40900), ], ) -def test_parse_coordinates(raw_coord: T.Optional[str], expected: T.Optional[float]): - assert _parse_coord(raw_coord) == pytest.approx(expected) +def test_parse_coordinates( + raw_coord: T.Optional[str], raw_ref: str, expected: T.Optional[float] +): + assert _parse_coord(raw_coord, raw_ref) == pytest.approx(expected) # test ExifWrite write a timestamp and ExifRead read it back