diff --git a/cartiflette/download/download.py b/cartiflette/download/download.py index 3ce104db..d84f86fb 100644 --- a/cartiflette/download/download.py +++ b/cartiflette/download/download.py @@ -11,6 +11,7 @@ from pebble import ThreadPool import s3fs import shutil +import traceback from typing import Union from cartiflette import BUCKET, PATH_WITHIN_BUCKET, FS, THREADS_DOWNLOAD @@ -280,7 +281,8 @@ def func(args): else: paths = {} # cleanup temp files - shutil.rmtree(result["root_cleanup"]) + if result["root_cleanup"]: + shutil.rmtree(result["root_cleanup"]) del result["hash"], result["root_cleanup"], result["layers"] result["paths"] = paths @@ -303,6 +305,7 @@ def func(args): break except Exception as e: logger.error(e) + logger.error(traceback.format_exc()) else: for args in combinations: files = deep_dict_update(files, func(args)) diff --git a/tests/mockups.py b/tests/mockups.py index 04dd3490..50f67b8c 100644 --- a/tests/mockups.py +++ b/tests/mockups.py @@ -2,6 +2,7 @@ import pytest import requests from requests_cache import CachedSession +import s3fs import logging from tests.conftest import ( @@ -10,7 +11,9 @@ CONTENT_DUMMY, ) +from cartiflette import FS from cartiflette.download.dataset import Dataset +from cartiflette.download.scraper import MasterScraper logging.basicConfig(level=logging.INFO) @@ -19,31 +22,28 @@ @pytest.fixture def mock_Dataset_without_s3(monkeypatch): monkeypatch.setattr(Dataset, "_get_last_md5", lambda x: None) - monkeypatch.setattr("FS") + # monkeypatch.setattr("FS") @pytest.fixture def total_mock_s3(monkeypatch): monkeypatch.setattr(Dataset, "_get_last_md5", lambda x: None) - def mock_unpack(x): + def mock_unpack(self, x): return { - x.provider: { - x.dataset_family: { - x.source: { - x.territory: { - x.year: { - "downloaded": False, - "paths": None, - "hash": None, - } - } - } - } - } + "downloaded": False, + "layers": None, + "hash": None, + "root_cleanup": None, } - monkeypatch.setattr(Dataset, "download_unpack", lambda x: mock_unpack) + monkeypatch.setattr(MasterScraper, "download_unpack", mock_unpack) + monkeypatch.setattr("cartiflette.THREADS_DOWNLOAD", 1) + + def mock_ls(folder): + return [f"{folder}/md5.json"] + + monkeypatch.setattr(FS, "ls", mock_ls) class MockResponse: diff --git a/tests/test_download.py b/tests/test_download.py index 687b5b59..031f0407 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -4,16 +4,18 @@ import os import requests_cache import logging + from cartiflette.download.dataset import Dataset from cartiflette.download.scraper import MasterScraper - from cartiflette.download.download import _download_sources +from cartiflette.download import download_all from cartiflette.utils import import_yaml_config from tests.conftest import ( DUMMY_FILE_1, DUMMY_FILE_2, HASH_DUMMY, ) + from tests.mockups import ( mock_httpscraper_download_success, mock_httpscraper_download_success_corrupt_hash, @@ -285,5 +287,6 @@ def test_sources_yaml(mock_Dataset_without_s3): def test_download_all(total_mock_s3): - ret = _download_sources() + ret = download_all() assert isinstance(ret, dict) + assert len(ret) > 0