Skip to content

Commit

Permalink
Disable progress bar for small fetchings (#1638)
Browse files Browse the repository at this point in the history
  • Loading branch information
AleksanderWWW authored Feb 8, 2024
1 parent 5c46070 commit 6d65cea
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 9 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
- Changed internal directories path structure ([#1606](https://github.com/neptune-ai/neptune-client/pull/1606))
- Changed format of warning messages ([#1635](https://github.com/neptune-ai/neptune-client/pull/1635))
- Make `trash_objects()` raise `ProjectNotFound` if project does not exist ([#1636](https://github.com/neptune-ai/neptune-client/pull/1636))
- Do not show progress bars when no data to fetch / small amount of data ([#1638](https://github.com/neptune-ai/neptune-client/pull/1638))


## 1.8.6
Expand Down
14 changes: 7 additions & 7 deletions src/neptune/api/searching_entries.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,19 +154,19 @@ def iter_over_pages(
searching_after = None
last_page = None

total = 0
total = get_single_page(
limit=0,
offset=0,
**kwargs,
).get("matchingItemCount", 0)

progress_bar = progress_bar if step_size >= total else None
progress_bar = False if total <= step_size else progress_bar # disable progress bar if only one page is fetched

with construct_progress_bar(progress_bar, "Fetching table...") as bar:
# beginning of the first page
bar.update(
by=0,
total=get_single_page(
limit=0,
offset=0,
**kwargs,
).get("matchingItemCount", 0),
total=total,
)

while True:
Expand Down
2 changes: 2 additions & 0 deletions src/neptune/attributes/series/fetchable_series.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ def make_row(entry: Row) -> Dict[str, Union[str, float, datetime]]:
row["timestamp"] = datetime.fromtimestamp(entry.timestampMillis / 1000)
return row

progress_bar = False if len(data) < limit else progress_bar

path = path_to_str(self._path) if hasattr(self, "_path") else ""
with construct_progress_bar(progress_bar, f"Fetching {path} values") as bar:
bar.update(by=len(data), total=val.totalItemCount) # first fetch before the loop
Expand Down
11 changes: 9 additions & 2 deletions src/neptune/internal/backends/hosted_file_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -473,21 +473,28 @@ def _store_response_as_file(
destination: Optional[str] = None,
progress_bar: Optional[ProgressBarType] = None,
) -> None:
chunk_size = 1024 * 1024

if destination is None:
target_file = _get_content_disposition_filename(response)
elif os.path.isdir(destination):
target_file = os.path.join(destination, _get_content_disposition_filename(response))
else:
target_file = destination

total_size = int(response.headers.get("content-length", 0))
if "content-length" in response.headers:
total_size = int(response.headers["content-length"])
progress_bar = False if total_size < chunk_size else progress_bar # less than one chunk
else:
total_size = 0

# TODO: update syntax once py3.10 becomes min supported version (with (x(), y(), z()): ...)
with ExitStack() as stack:
bar = stack.enter_context(construct_progress_bar(progress_bar, "Fetching file..."))
response = stack.enter_context(response)
file_stream = stack.enter_context(open(target_file, "wb"))

for chunk in response.iter_content(chunk_size=1024 * 1024):
for chunk in response.iter_content(chunk_size=chunk_size):
if chunk:
file_stream.write(chunk)
bar.update(by=len(chunk), total=total_size)
Expand Down

0 comments on commit 6d65cea

Please sign in to comment.