Skip to content

Commit

Permalink
[Streaming] retry on requests errors (#6963)
Browse files Browse the repository at this point in the history
* [Streaming] retry on requests errors

* lucain's comment
  • Loading branch information
lhoestq authored and albertvillanova committed Aug 14, 2024
1 parent fc96e93 commit 279c5bd
Showing 1 changed file with 8 additions and 3 deletions.
11 changes: 8 additions & 3 deletions src/datasets/utils/file_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
Copyright by the AllenNLP authors.
"""

import asyncio
import copy
import glob
import io
Expand All @@ -20,7 +21,6 @@
import warnings
import xml.dom.minidom
import zipfile
from asyncio import TimeoutError
from contextlib import closing, contextmanager
from functools import partial
from io import BytesIO
Expand All @@ -31,10 +31,10 @@
from urllib.parse import urljoin, urlparse
from xml.etree import ElementTree as ET

import aiohttp.client_exceptions
import fsspec
import huggingface_hub
import requests
from aiohttp.client_exceptions import ClientError
from fsspec.core import strip_protocol, url_to_fs
from fsspec.utils import can_be_local
from huggingface_hub.utils import EntryNotFoundError, insecure_hashlib
Expand Down Expand Up @@ -1103,7 +1103,12 @@ def read_with_retries(*args, **kwargs):
try:
out = read(*args, **kwargs)
break
except (ClientError, TimeoutError) as err:
except (
aiohttp.client_exceptions.ClientError,
asyncio.TimeoutError,
requests.exceptions.ConnectTimeout,
requests.exceptions.ConnectionError,
) as err:
disconnect_err = err
logger.warning(
f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RETRY_INTERVAL}sec [{retry}/{max_retries}]"
Expand Down

0 comments on commit 279c5bd

Please sign in to comment.