Skip to content
This repository has been archived by the owner on Oct 9, 2023. It is now read-only.

Add download support for tar.gz & don't download data if exists #157

Merged
merged 16 commits into from
Mar 22, 2021
Merged
16 changes: 8 additions & 8 deletions flash/core/data/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,16 @@ def download_file(url: str, path: str, verbose: bool = False) -> None:
if not os.path.exists(path):
os.makedirs(path)
local_filename = os.path.join(path, url.split('/')[-1])
r = requests.get(url, stream=True)
file_size = int(r.headers['Content-Length']) if 'Content-Length' in r.headers else 0
chunk = 1
chunk_size = 1024
num_bars = int(file_size / chunk_size)
if verbose:
print(dict(file_size=file_size))
print(dict(num_bars=num_bars))

if not os.path.exists(local_filename):
r = requests.get(url, stream=True)
file_size = int(r.headers['Content-Length']) if 'Content-Length' in r.headers else 0
kaushikb11 marked this conversation as resolved.
Show resolved Hide resolved
chunk = 1
chunk_size = 1024
num_bars = int(file_size / chunk_size)
if verbose:
print(dict(file_size=file_size))
print(dict(num_bars=num_bars))
kaushikb11 marked this conversation as resolved.
Show resolved Hide resolved
with open(local_filename, 'wb') as fp:
for chunk in tq(
r.iter_content(chunk_size=chunk_size),
Expand Down