From 3ac08965fd82ad25ecd3bb8983a66961a2fe2499 Mon Sep 17 00:00:00 2001 From: ThibaultFy Date: Thu, 29 Aug 2024 11:00:15 +0200 Subject: [PATCH 1/7] chore: remove tqdm Signed-off-by: ThibaultFy --- backend/image_transfer/encoder.py | 11 ++++------- backend/requirements.txt | 1 - 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/backend/image_transfer/encoder.py b/backend/image_transfer/encoder.py index 9977ddae9..e2f6b6d06 100644 --- a/backend/image_transfer/encoder.py +++ b/backend/image_transfer/encoder.py @@ -10,7 +10,6 @@ from dxf import DXF from dxf import DXFBase from requests import HTTPError -from tqdm import tqdm from image_transfer.common import Authenticator from image_transfer.common import Blob @@ -61,12 +60,10 @@ def download_blob_to_zip(dxf_base: DXFBase, blob: Blob, zip_file: safezip.ZipFil bytes_iterator, total_size = repository_dxf.pull_blob(blob.digest, size=True) # we write the blob directly to the zip file - with tqdm(total=total_size, unit="B", unit_scale=True) as pbar: - blob_path_in_zip = f"blobs/{blob.digest}" - with zip_file.open(blob_path_in_zip, "w", force_zip64=True) as blob_in_zip: - for chunk in bytes_iterator: - blob_in_zip.write(chunk) - pbar.update(len(chunk)) + blob_path_in_zip = f"blobs/{blob.digest}" + with zip_file.open(blob_path_in_zip, "w", force_zip64=True) as blob_in_zip: + for chunk in bytes_iterator: + blob_in_zip.write(chunk) return blob_path_in_zip diff --git a/backend/requirements.txt b/backend/requirements.txt index 3b8a16c18..c5ab6e394 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -24,7 +24,6 @@ django-filter==24.2 pydantic==2.4.0 redis==5.0.0 mozilla-django-oidc==4.0.1 -tqdm==4.66.4 python-dxf==12.1.0 watchdog==2.1.9 # Prevent error linked to Deprecation warnings From 5968d209c815363d37439cebeac8d01a1dc3cb70 Mon Sep 17 00:00:00 2001 From: ThibaultFy Date: Thu, 29 Aug 2024 11:01:18 +0200 Subject: [PATCH 2/7] chore: changelog Signed-off-by: ThibaultFy --- changes/990.removed | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/990.removed diff --git a/changes/990.removed b/changes/990.removed new file mode 100644 index 000000000..e46dcafbc --- /dev/null +++ b/changes/990.removed @@ -0,0 +1 @@ +`tqdm`dependency, only used in logging From d4133bfa97860912d9f85215b90e11af09797967 Mon Sep 17 00:00:00 2001 From: ThibaultFy Date: Thu, 29 Aug 2024 11:06:42 +0200 Subject: [PATCH 3/7] chore: add logging Signed-off-by: ThibaultFy --- backend/image_transfer/encoder.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/image_transfer/encoder.py b/backend/image_transfer/encoder.py index e2f6b6d06..93d439dbe 100644 --- a/backend/image_transfer/encoder.py +++ b/backend/image_transfer/encoder.py @@ -7,6 +7,7 @@ from typing import Optional from typing import Union +import structlog from dxf import DXF from dxf import DXFBase from requests import HTTPError @@ -22,6 +23,8 @@ from substrapp.docker_registry import RegistryPreconditionFailedException from substrapp.utils import safezip +logger = structlog.get_logger("events") + def add_blobs_to_zip( dxf_base: DXFBase, @@ -64,6 +67,8 @@ def download_blob_to_zip(dxf_base: DXFBase, blob: Blob, zip_file: safezip.ZipFil with zip_file.open(blob_path_in_zip, "w", force_zip64=True) as blob_in_zip: for chunk in bytes_iterator: blob_in_zip.write(chunk) + + logger.info(f"Blob {blob} of size {total_size} downloaded and stored in zip file") return blob_path_in_zip From 4727fb668099ae686550f45dad8333076478cc0e Mon Sep 17 00:00:00 2001 From: ThibaultFy Date: Mon, 2 Sep 2024 17:18:42 +0200 Subject: [PATCH 4/7] chore: wrap function in try expects with logs Signed-off-by: ThibaultFy --- backend/image_transfer/encoder.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/backend/image_transfer/encoder.py b/backend/image_transfer/encoder.py index 93d439dbe..d3dee15cb 100644 --- a/backend/image_transfer/encoder.py +++ b/backend/image_transfer/encoder.py @@ -60,13 +60,21 @@ def add_blobs_to_zip( def download_blob_to_zip(dxf_base: DXFBase, blob: Blob, zip_file: safezip.ZipFile): repository_dxf = DXF.from_base(dxf_base, blob.repository) - bytes_iterator, total_size = repository_dxf.pull_blob(blob.digest, size=True) + try: + bytes_iterator, total_size = repository_dxf.pull_blob(blob.digest, size=True) + except Exception as e: + logger.exception(f"Failed to download blob {blob}", e=e) + raise e # we write the blob directly to the zip file blob_path_in_zip = f"blobs/{blob.digest}" - with zip_file.open(blob_path_in_zip, "w", force_zip64=True) as blob_in_zip: - for chunk in bytes_iterator: - blob_in_zip.write(chunk) + try: + with zip_file.open(blob_path_in_zip, "w", force_zip64=True) as blob_in_zip: + for chunk in bytes_iterator: + blob_in_zip.write(chunk) + except Exception as e: + logger.exception(f"Failed to write blob {blob} to zip file", e=e) + raise e logger.info(f"Blob {blob} of size {total_size} downloaded and stored in zip file") return blob_path_in_zip From b369be7a958261cdf0e8be5c571785db04757565 Mon Sep 17 00:00:00 2001 From: ThibaultFy Date: Wed, 4 Sep 2024 11:24:58 +0200 Subject: [PATCH 5/7] chore: fix event logger Signed-off-by: ThibaultFy --- backend/image_transfer/encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/image_transfer/encoder.py b/backend/image_transfer/encoder.py index d3dee15cb..4adfb2e2e 100644 --- a/backend/image_transfer/encoder.py +++ b/backend/image_transfer/encoder.py @@ -23,7 +23,7 @@ from substrapp.docker_registry import RegistryPreconditionFailedException from substrapp.utils import safezip -logger = structlog.get_logger("events") +logger = structlog.get_logger("builder") def add_blobs_to_zip( From 1bf0d35e0cf039d085fa1e906dd020dd17c0e4c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilhem=20Barth=C3=A9s?= Date: Fri, 13 Sep 2024 14:57:02 +0200 Subject: [PATCH 6/7] chore: change logger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Guilhem Barthés --- backend/image_transfer/encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/image_transfer/encoder.py b/backend/image_transfer/encoder.py index 4adfb2e2e..1718e1a4f 100644 --- a/backend/image_transfer/encoder.py +++ b/backend/image_transfer/encoder.py @@ -23,7 +23,7 @@ from substrapp.docker_registry import RegistryPreconditionFailedException from substrapp.utils import safezip -logger = structlog.get_logger("builder") +logger = structlog.get_logger("worker") def add_blobs_to_zip( From fd4a3e41ed11b90f2cc0105cc9ffa6866f557776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilhem=20Barth=C3=A9s?= Date: Fri, 13 Sep 2024 14:58:10 +0200 Subject: [PATCH 7/7] chore: typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Guilhem Barthés --- changes/990.removed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/990.removed b/changes/990.removed index e46dcafbc..86af537ad 100644 --- a/changes/990.removed +++ b/changes/990.removed @@ -1 +1 @@ -`tqdm`dependency, only used in logging +`tqdm` dependency, only used in logging