diff --git a/esmvalcore/esgf/_download.py b/esmvalcore/esgf/_download.py index 551aacceed..445ff7a1d8 100644 --- a/esmvalcore/esgf/_download.py +++ b/esmvalcore/esgf/_download.py @@ -466,7 +466,10 @@ def _download(self, local_file, url): cert=get_credentials()) response.raise_for_status() with tmp_file.open("wb") as file: - for chunk in response.iter_content(chunk_size=None): + # Specify chunk_size to avoid + # https://github.com/psf/requests/issues/5536 + megabyte = 2**20 + for chunk in response.iter_content(chunk_size=megabyte): if hasher is not None: hasher.update(chunk) file.write(chunk) diff --git a/tests/unit/esgf/test_download.py b/tests/unit/esgf/test_download.py index 3b0030694d..6ba258fac0 100644 --- a/tests/unit/esgf/test_download.py +++ b/tests/unit/esgf/test_download.py @@ -459,7 +459,7 @@ def test_single_download(mocker, tmp_path, checksum): # We checked for a valid response response.raise_for_status.assert_called_once() # And requested a reasonable chunk size - response.iter_content.assert_called_with(chunk_size=None) + response.iter_content.assert_called_with(chunk_size=2**20) def test_download_skip_existing(tmp_path, caplog):