From e540d3d3525358c3649e2958152a7485c0d5d5b7 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 11 Dec 2024 09:47:34 +0100 Subject: [PATCH 1/5] edit --- .../ingest_xmlupload/bulk_ingest_client.py | 32 +++---------------- .../upload_files/upload_files.py | 4 +-- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py index 5f85eabb8..96072bf6a 100644 --- a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +++ b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py @@ -57,35 +57,11 @@ def __post_init__(self) -> None: def upload_file( self, filepath: Path, - ) -> UploadFailure | None: + ) -> None: """Uploads a file to the ingest server.""" - try: - with open(self.imgdir / filepath, "rb") as binary_io: - content = binary_io.read() - except OSError as e: - err_msg = f"Cannot bulk-ingest {filepath}, because the file could not be opened/read: {e.strerror}" - logger.error(err_msg) - return UploadFailure(filepath, err_msg) - url = self._build_url_for_bulk_ingest_ingest_route(filepath) - headers = {"Content-Type": "application/octet-stream"} - timeout = 600 - err_msg = f"Failed to upload '{filepath}' to '{url}'." - try: - logger.debug(f"REQUEST: POST to {url}, timeout: {timeout}, headers: {headers}") - res = self.session.post( - url=url, - headers=headers, - data=content, - timeout=timeout, - ) - logger.debug(f"RESPONSE: {res.status_code}") - except RequestException as e: - logger.error(err_msg) - return UploadFailure(filepath, f"Exception of requests library: {e}") - if res.status_code != STATUS_OK: - logger.error(err_msg) - return UploadFailure(filepath, res.reason, res.status_code, res.text) - return None + filepath_rel = filepath.relative_to("/") if filepath.is_absolute() else filepath + target_base = Path("/Volumes/LHTT/dsp-api/sipi/tmp-dsp-ingest/import/0820") + (target_base / filepath_rel).symlink_to(filepath) def _build_url_for_bulk_ingest_ingest_route(self, filepath: Path) -> str: """ diff --git a/src/dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py b/src/dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py index 6a8170f9e..61ff626bf 100644 --- a/src/dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py +++ b/src/dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py @@ -42,9 +42,7 @@ def upload_files( failures: list[UploadFailure] = [] progress_bar = tqdm(paths, desc="Uploading files", unit="file(s)", dynamic_ncols=True) for path in progress_bar: - if res := ingest_client.upload_file(path): - failures.append(res) - progress_bar.set_description(f"Uploading files (failed: {len(failures)})") + ingest_client.upload_file(path) if failures: aggregated_failures = UploadFailures(failures, len(paths), shortcode, creds.dsp_ingest_url) msg = aggregated_failures.execute_error_protocol() From e0016f8ea33ca63cd0bf6111c4474d9f6bb6d533 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 11 Dec 2024 11:49:35 +0100 Subject: [PATCH 2/5] fix --- .../commands/ingest_xmlupload/bulk_ingest_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py index 96072bf6a..03231039a 100644 --- a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +++ b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py @@ -60,8 +60,10 @@ def upload_file( ) -> None: """Uploads a file to the ingest server.""" filepath_rel = filepath.relative_to("/") if filepath.is_absolute() else filepath - target_base = Path("/Volumes/LHTT/dsp-api/sipi/tmp-dsp-ingest/import/0820") - (target_base / filepath_rel).symlink_to(filepath) + link_source_base = Path("/Volumes/LHTT/dsp-api/sipi/tmp-dsp-ingest/import/0820") + link_source = link_source_base / filepath_rel + link_source.parent.mkdir(parents=True, exist_ok=True) + link_source.symlink_to(filepath) def _build_url_for_bulk_ingest_ingest_route(self, filepath: Path) -> str: """ From 20f23ca9ad58f63f9f716717ab323d062bdc0c75 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 11 Dec 2024 13:03:38 +0100 Subject: [PATCH 3/5] hardlink --- src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py index 03231039a..f4847a61d 100644 --- a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +++ b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py @@ -63,7 +63,7 @@ def upload_file( link_source_base = Path("/Volumes/LHTT/dsp-api/sipi/tmp-dsp-ingest/import/0820") link_source = link_source_base / filepath_rel link_source.parent.mkdir(parents=True, exist_ok=True) - link_source.symlink_to(filepath) + link_source.hardlink_to(filepath) def _build_url_for_bulk_ingest_ingest_route(self, filepath: Path) -> str: """ From 078b2a8048458c762f240679155698ef5854cb84 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Thu, 12 Dec 2024 09:38:33 +0100 Subject: [PATCH 4/5] exFAT doesn't support hardlinks -> copy instead --- .../commands/ingest_xmlupload/bulk_ingest_client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py index f4847a61d..8adceb920 100644 --- a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +++ b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py @@ -1,3 +1,4 @@ +import shutil import urllib.parse from dataclasses import dataclass from dataclasses import field @@ -60,10 +61,10 @@ def upload_file( ) -> None: """Uploads a file to the ingest server.""" filepath_rel = filepath.relative_to("/") if filepath.is_absolute() else filepath - link_source_base = Path("/Volumes/LHTT/dsp-api/sipi/tmp-dsp-ingest/import/0820") - link_source = link_source_base / filepath_rel - link_source.parent.mkdir(parents=True, exist_ok=True) - link_source.hardlink_to(filepath) + target_base = Path("/Volumes/LHTT/dsp-api/sipi/tmp-dsp-ingest/import/0820") + target = target_base / filepath_rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(filepath, target) def _build_url_for_bulk_ingest_ingest_route(self, filepath: Path) -> str: """ From 7b62d4bfd580ac42c7b8d878f06db067df7bb04b Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 21 Jan 2025 14:19:13 +0100 Subject: [PATCH 5/5] rename to Thoth --- src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py index aa7f311fe..16aa1536a 100644 --- a/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +++ b/src/dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py @@ -67,7 +67,7 @@ def upload_file( # noqa: DAR201 """ filepath_rel = filepath.relative_to("/") if filepath.is_absolute() else filepath - target_base = Path("/Volumes/Salsah/LHTT/dsp-api/sipi/tmp-dsp-ingest/import/0820") + target_base = Path("/Volumes/Thoth/dsp-api/sipi/tmp-dsp-ingest/import/0820") target = target_base / filepath_rel target.parent.mkdir(parents=True, exist_ok=True) shutil.copy(filepath, target)