diff --git a/Dockerfile b/Dockerfile index e75937a58..1087b55fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,18 @@ FROM alpine:3.18 AS runtime_amd64_none ENV MUSL_LOCPATH="/usr/share/i18n/locales/musl" -RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang +RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang exiftool WORKDIR /app COPY dist/icloudpd-ex-*.*.*-linux-musl-amd64 icloudpd_ex FROM alpine:3.18 AS runtime_arm64_none ENV MUSL_LOCPATH="/usr/share/i18n/locales/musl" -RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang +RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang exiftool WORKDIR /app COPY dist/icloudpd-ex-*.*.*-linux-musl-arm64 icloudpd_ex FROM alpine:3.18 AS runtime_arm_v7 ENV MUSL_LOCPATH="/usr/share/i18n/locales/musl" -RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang +RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang exiftool WORKDIR /app COPY dist/icloudpd-ex-*.*.*-linux-musl-arm32v7 icloudpd_ex diff --git a/pyproject.toml b/pyproject.toml index 18b270eb9..685c421ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ dependencies = [ "click==8.1.6", "tqdm==4.66.4", "piexif==1.1.3", + "pyexiftool==0.5.6", + "python-dateutil==2.9.0.post0", "urllib3==1.26.16", "typing_extensions==4.11.0", "Flask==3.0.3", @@ -68,6 +70,7 @@ test = [ "pytest-timeout==2.1.0", "pytest-xdist==3.3.1", "mypy==1.10.1", + "types-python-dateutil==2.9.0.20241003", "types-pytz==2024.1.0.20240417", "types-tzlocal==5.1.0.1", "types-requests==2.31.0.2", @@ -103,7 +106,7 @@ where = ["src"] # list of folders that contain the packages (["."] by default) exclude = ["starters"] [[tool.mypy.overrides]] -module = ['piexif.*', 'vcr.*', 'srp.*'] +module = ['piexif.*', 'vcr.*', 'srp.*', 'exiftool.*'] ignore_missing_imports = true [tool.ruff] diff --git a/src/icloudpd/autodelete.py b/src/icloudpd/autodelete.py index 018e0637d..5937bc940 100644 --- a/src/icloudpd/autodelete.py +++ b/src/icloudpd/autodelete.py @@ -73,9 +73,11 @@ def autodelete_photos( for _size, _version in disambiguate_filenames(media.versions, _sizes).items(): if _size in [AssetVersionSize.ALTERNATIVE, AssetVersionSize.ADJUSTED]: paths.add(os.path.normpath(local_download_path(_version.filename, download_dir))) + paths.add(os.path.normpath(local_download_path(_version.filename, download_dir)) + ".xmp") for _size, _version in media.versions.items(): if _size not in [AssetVersionSize.ALTERNATIVE, AssetVersionSize.ADJUSTED]: paths.add(os.path.normpath(local_download_path(_version.filename, download_dir))) + paths.add(os.path.normpath(local_download_path(_version.filename, download_dir)) + ".xmp") for path in paths: if os.path.exists(path): logger.debug("Deleting %s...", path) diff --git a/src/icloudpd/base.py b/src/icloudpd/base.py index 319dffa58..de15bad39 100644 --- a/src/icloudpd/base.py +++ b/src/icloudpd/base.py @@ -64,6 +64,7 @@ from icloudpd.server import serve_app from icloudpd.status import Status, StatusExchange from icloudpd.string_helpers import truncate_middle +from icloudpd.xmp_sidecar import generate_xmp_file, init_exiftool def build_filename_cleaner( @@ -377,6 +378,11 @@ def report_version(ctx: click.Context, _param: click.Parameter, value: bool) -> help="Don't download any live photos (default: Download live photos)", is_flag=True, ) +@click.option( + "--xmp-sidecar", + help="Export additional data as XMP sidecar files (default: don't export)", + is_flag=True, +) @click.option( "--force-size", help="Only download the requested size (`adjusted` and `alternate` will not be forced)" @@ -583,6 +589,7 @@ def main( list_libraries: bool, skip_videos: bool, skip_live_photos: bool, + xmp_sidecar: bool, force_size: bool, auto_delete: bool, only_print_filenames: bool, @@ -694,6 +701,7 @@ def main( list_libraries=list_libraries, skip_videos=skip_videos, skip_live_photos=skip_live_photos, + xmp_sidecar=xmp_sidecar, force_size=force_size, auto_delete=auto_delete, only_print_filenames=only_print_filenames, @@ -742,6 +750,10 @@ def main( server_thread = Thread(target=serve_app, daemon=True, args=[logger, status_exchange]) server_thread.start() + # initialize ExifTool + if xmp_sidecar: + init_exiftool(logger=logging.getLogger(__name__)) + result = core( download_builder( logger, @@ -756,6 +768,7 @@ def main( live_photo_size, dry_run, file_match_policy, + xmp_sidecar ) if directory is not None else (lambda _s: lambda _c, _p: False), @@ -812,6 +825,7 @@ def download_builder( live_photo_size: LivePhotoVersionSize, dry_run: bool, file_match_policy: FileMatchPolicy, + xmp_sidecar: bool ) -> Callable[[PyiCloudService], Callable[[Counter, PhotoAsset], bool]]: """factory for downloader""" @@ -960,6 +974,9 @@ def download_photo_(counter: Counter, photo: PhotoAsset) -> bool: download.set_utime(download_path, created_date) logger.info("Downloaded %s", truncated_path) + if xmp_sidecar: + generate_xmp_file(logger, download_path, photo._asset_record) + # Also download the live photo if present if not skip_live_photos: lp_size = live_photo_size diff --git a/src/icloudpd/config.py b/src/icloudpd/config.py index 4a25203e2..cbede5d56 100644 --- a/src/icloudpd/config.py +++ b/src/icloudpd/config.py @@ -24,6 +24,7 @@ def __init__( list_libraries: bool, skip_videos: bool, skip_live_photos: bool, + xmp_sidecar: bool, force_size: bool, auto_delete: bool, only_print_filenames: bool, diff --git a/src/icloudpd/xmp_sidecar.py b/src/icloudpd/xmp_sidecar.py new file mode 100644 index 000000000..0765e15c7 --- /dev/null +++ b/src/icloudpd/xmp_sidecar.py @@ -0,0 +1,79 @@ +"""Generate XMP sidecar file from photo asset record""" + +from __future__ import annotations + +import base64 +import logging +import plistlib +from datetime import datetime +from typing import Any + +import dateutil.tz +from exiftool import ExifToolHelper + +exif_tool = None +def init_exiftool(logger: logging.Logger) -> None: + """Initialize ExifTool""" + global exif_tool + if not exif_tool: + try: + exif_tool = ExifToolHelper(logger=logging.getLogger(__name__)) + except FileNotFoundError as err: + logger.warning(err) + logger.warning("XMP sidecar files will not be generated") + else: + raise Exception("ExifTool already initialized") + +def build_exiftool_arguments(asset_record: dict[str, Any]) -> list[str]: + xmp_metadata: dict[str, str|int] = {} + if 'captionEnc' in asset_record['fields']: + xmp_metadata['Title'] = base64.b64decode(asset_record['fields']['captionEnc']['value']).decode('utf-8') + if 'extendedDescEnc' in asset_record['fields']: + xmp_metadata['Description'] = base64.b64decode(asset_record['fields']['extendedDescEnc']['value']).decode('utf-8') + if 'orientation' in asset_record['fields']: + xmp_metadata['Orientation'] = asset_record['fields']['orientation']['value'] + if 'assetSubtypeV2' in asset_record['fields'] and int(asset_record['fields']['assetSubtypeV2']['value']) == 3: + xmp_metadata["Make"] = "Screenshot" + xmp_metadata["DigitalSourceType"] = "screenCapture" + if 'keywordsEnc' in asset_record['fields']: + keywords = plistlib.loads(base64.b64decode(asset_record['fields']['keywordsEnc']['value']), fmt=plistlib.FMT_BINARY) + if(len(keywords) > 0): + xmp_metadata["IPTC:keywords"] = ",".join(keywords) + if 'locationEnc' in asset_record['fields']: + locationDec = plistlib.loads(base64.b64decode(asset_record['fields']['locationEnc']['value']), fmt=plistlib.FMT_BINARY) + if('alt' in locationDec): + xmp_metadata["GPSAltitude"] = locationDec['alt'] + if('lat' in locationDec): + xmp_metadata["GPSLatitude"] = locationDec['lat'] + if('lon' in locationDec): + xmp_metadata["GPSLongitude"] = locationDec['lon'] + if('speed' in locationDec): + xmp_metadata["GPSSpeed"] = locationDec['speed'] + if('timestamp' in locationDec and isinstance(locationDec['timestamp'], datetime)): + xmp_metadata["exif:GPSDateTime"] = locationDec['timestamp'].strftime("%Y:%m:%d %H:%M:%S.%f%z") + if 'assetDate' in asset_record['fields']: + timeZoneOffset = 0 + if timeZoneOffset in asset_record['fields']: + timeZoneOffset = int(asset_record['fields']['timeZoneOffset']['value']) + assetDate = datetime.fromtimestamp(int(asset_record['fields']['assetDate']['value'])/1000,tz=dateutil.tz.tzoffset(None, timeZoneOffset)) + assetDateString = assetDate.strftime("%Y:%m:%d %H:%M:%S.%f%z") + assetDateString = f"{assetDateString[:-2]}:{assetDateString[-2:]}" # Add a colon to timezone offset + xmp_metadata["XMP-photoshop:DateCreated"] = assetDateString # Apple Photos uses this field when exporting an XMP sidecar + xmp_metadata["CreateDate"] = assetDateString + # Hidden or Deleted Photos should be marked as rejected (needs running as --album "Hidden" or --album "Recently Deleted") + if (('isHidden' in asset_record['fields'] and asset_record['fields']['isHidden']['value'] == 1) or + ('isDeleted' in asset_record['fields'] and asset_record['fields']['isDeleted']['value'] == 1)): + # -1 means rejected: https://www.iptc.org/std/photometadata/specification/IPTC-PhotoMetadata#image-rating + xmp_metadata["Rating"] = -1 + elif asset_record['fields']['isFavorite']['value'] == 1: #only mark photo as favorite if not hidden or deleted + xmp_metadata["Rating"] = 5 + + args = ["-" + k + "=" + str(xmp_metadata[k]) for k in xmp_metadata] + return args + +def generate_xmp_file(logger: logging.Logger, download_path: str, asset_record: dict[str, Any]) -> None: + """Generate XMP sidecar file from photo asset record""" + if exif_tool: + args = build_exiftool_arguments(asset_record) + # json.dump(asset_record['fields'], open(download_path+".ar.json", "w"), indent=4) + exif_tool.execute("-overwrite_original", download_path+".xmp", *args) diff --git a/src/pyicloud_ipd/services/photos.py b/src/pyicloud_ipd/services/photos.py index 7322ba31c..6eec1b7bf 100644 --- a/src/pyicloud_ipd/services/photos.py +++ b/src/pyicloud_ipd/services/photos.py @@ -509,6 +509,7 @@ def _list_query_gen(self, offset: int, list_type: str, direction: str, query_fil u'locationLatitude', u'locationLongitude', u'adjustmentType', u'timeZoneOffset', u'vidComplDurValue', u'vidComplDurScale', u'vidComplDispValue', u'vidComplDispScale', + u'keywordsEnc',u'extendedDescEnc', u'vidComplVisibilityState', u'customRenderedValue', u'containerId', u'itemId', u'position', u'isKeyAsset' ], diff --git a/tests/test_xmp_sidecar.py b/tests/test_xmp_sidecar.py new file mode 100644 index 000000000..14bc997cd --- /dev/null +++ b/tests/test_xmp_sidecar.py @@ -0,0 +1,90 @@ +from typing import Any, Dict +from unittest import TestCase + +from icloudpd.xmp_sidecar import build_exiftool_arguments + + +class BuildExifToolArguments(TestCase): + def test_build_exiftool_arguments(self) -> None: + assetRecordStub: Dict[str,Dict[str,Any]] = { + 'fields': { + "captionEnc": { + "value": "VGl0bGUgSGVyZQ==", + "type": "ENCRYPTED_BYTES" + }, + "extendedDescEnc": { + "value": "Q2FwdGlvbiBIZXJl", + "type": "ENCRYPTED_BYTES" + }, + 'orientation': { + "value" : 6, + "type" : "INT64" + }, + 'assetSubtypeV2' : { + "value" : 2, + "type" : "INT64" + }, + "keywordsEnc": { + "value": "YnBsaXN0MDChAVxzb21lIGtleXdvcmQICgAAAAAAAAEBAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAX", + "type": "ENCRYPTED_BYTES" + }, + 'locationEnc': { + "value" : "YnBsaXN0MDDYAQIDBAUGBwgJCQoLCQwNCVZjb3Vyc2VVc3BlZWRTYWx0U2xvbld2ZXJ0QWNjU2xhdFl0aW1lc3RhbXBXaG9yekFjYyMAAAAAAAAAACNAdG9H6P0fpCNAWL2oZnRhiiNAMtKmTC+DezMAAAAAAAAAAAgZICYqLjY6RExVXmdwAAAAAAAAAQEAAAAAAAAADgAAAAAAAAAAAAAAAAAAAHk=", + "type" : "ENCRYPTED_BYTES" + }, + 'assetDate' : { + "value" : 1532951050176, + "type" : "TIMESTAMP" + }, + 'isHidden': { + "value" : 0, + "type" : "INT64" + }, + 'isDeleted': { + "value" : 0, + "type" : "INT64" + }, + 'isFavorite': { + "value" : 0, + "type" : "INT64" + }, + }, + } + + # Test full stub record + args = build_exiftool_arguments(assetRecordStub) + self.assertCountEqual(args , [ + '-Title=Title Here', + '-Description=Caption Here', + '-IPTC:keywords=some keyword', + '-GPSAltitude=326.9550561797753', + '-GPSLatitude=18.82285', + '-GPSLongitude=98.96340333333333', + '-GPSSpeed=0.0', + '-exif:GPSDateTime=2001:01:01 00:00:00.000000', + '-XMP-photoshop:DateCreated=2018:07:30 11:44:10.176000+00:00', + '-CreateDate=2018:07:30 11:44:10.176000+00:00', + '-Orientation=6' + ]) + + # Test Screenshot Tagging + assetRecordStub['fields']['assetSubtypeV2']['value'] = 3 + args = build_exiftool_arguments(assetRecordStub) + assert "-Make=Screenshot" in args + assert "-DigitalSourceType=screenCapture" in args + + # Test Favorites + assetRecordStub['fields']['isFavorite']['value'] = 1 + args = build_exiftool_arguments(assetRecordStub) + assert "-Rating=5" in args + + # Test Deleted + assetRecordStub['fields']['isDeleted']['value'] = 1 + args = build_exiftool_arguments(assetRecordStub) + assert "-Rating=-1" in args + + # Test Hidden + assetRecordStub['fields']['isDeleted']['value'] = 0 + assetRecordStub['fields']['isHidden']['value'] = 1 + args = build_exiftool_arguments(assetRecordStub) + assert "-Rating=-1" in args