Skip to content

Commit

Permalink
Issue #96: ensure the created dimension is sorted (#101)
Browse files Browse the repository at this point in the history
* implement sorting of the output queue according to the order of the input queue to satisfy issue #96

* Update CHANGELOG.md with issue-96 fix
  • Loading branch information
ank1m authored Feb 14, 2024
1 parent 95892f2 commit 6fdeeb2
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 5 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Added
### Changed
- [issues/96](https://github.com/podaac/concise/issues/96):
- impelemented sorting after [multi_core_download](https://github.com/podaac/concise/blob/23b44803f4829c1eb7e9d39b311a0373092daab3/podaac/merger/harmony/download_worker.py#L15) to preserve the input file order in the
### Deprecated
### Removed
### Fixed
Expand Down
10 changes: 5 additions & 5 deletions podaac/merger/harmony/download_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ def multi_core_download(urls, destination_dir, access_token, cfg, process_count=
url_queue = manager.Queue(len(urls))
path_list = manager.list()

for url in urls:
url_queue.put(url)
for iurl, url in enumerate(urls):
url_queue.put((iurl, url))

# Spawn worker processes
processes = []
Expand All @@ -64,7 +64,7 @@ def multi_core_download(urls, destination_dir, access_token, cfg, process_count=

path_list = deepcopy(path_list) # ensure GC can cleanup multiprocessing

return [Path(path) for path in path_list]
return [Path(path) for ipath, path in sorted(path_list)]


def _download_worker(url_queue, path_list, destination_dir, access_token, cfg):
Expand All @@ -91,7 +91,7 @@ def _download_worker(url_queue, path_list, destination_dir, access_token, cfg):

while not url_queue.empty():
try:
url = url_queue.get_nowait()
iurl, url = url_queue.get_nowait()
except queue.Empty:
break

Expand All @@ -105,4 +105,4 @@ def _download_worker(url_queue, path_list, destination_dir, access_token, cfg):
else:
logger.warning('Origin filename could not be assertained - %s', url)

path_list.append(str(path))
path_list.append((iurl, str(path)))

0 comments on commit 6fdeeb2

Please sign in to comment.