From c942bbde3a1399cccd411dcc41354b38cf7a8c47 Mon Sep 17 00:00:00 2001 From: Sebastian Markgraf Date: Wed, 14 Dec 2022 13:57:23 +0000 Subject: [PATCH 01/12] Add toggle to add_samples to hide progress bar --- fiftyone/core/dataset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fiftyone/core/dataset.py b/fiftyone/core/dataset.py index 9afc213331..feee6ef296 100644 --- a/fiftyone/core/dataset.py +++ b/fiftyone/core/dataset.py @@ -2441,6 +2441,7 @@ def add_samples( dynamic=False, validate=True, num_samples=None, + progress=True, ): """Adds the given samples to the dataset. @@ -2462,6 +2463,7 @@ def add_samples( num_samples (None): the number of samples in ``samples``. If not provided, this is computed via ``len(samples)``, if possible. This value is optional and is used only for progress tracking + progress (True): whether to show the progress bar of the import Returns: a list of IDs of the samples in the dataset @@ -2473,7 +2475,7 @@ def add_samples( pass batcher = fou.get_default_batcher( - samples, progress=True, total=num_samples + samples, progress=progress, total=num_samples ) sample_ids = [] From 6652862707130ca0174544e1636be5eeaf3c6d7e Mon Sep 17 00:00:00 2001 From: brimoor Date: Sat, 22 Jul 2023 09:19:32 -0400 Subject: [PATCH 02/12] feat(progress): Add first progress bar estcase --- fiftyone/core/dataset.py | 46 ++++++++++++++++++++----------- fiftyone/core/metadata.py | 27 ++++++++++++++---- fiftyone/core/utils.py | 16 ++++++++--- fiftyone/zoo/datasets/__init__.py | 4 +++ tests/unittests/utils_tests.py | 38 +++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 26 deletions(-) diff --git a/fiftyone/core/dataset.py b/fiftyone/core/dataset.py index feee6ef296..b3fb934c2a 100644 --- a/fiftyone/core/dataset.py +++ b/fiftyone/core/dataset.py @@ -2130,7 +2130,7 @@ def delete_group_slice(self, name): self.save() - def iter_samples(self, progress=False, autosave=False, batch_size=None): + def iter_samples(self, progress=None, autosave=False, batch_size=None): """Returns an iterator over the samples in the dataset. Examples:: @@ -2164,7 +2164,7 @@ def make_label(): sample.ground_truth.label = make_label() Args: - progress (False): whether to render a progress bar tracking the + progress (None): whether to render a progress bar tracking the iterator's progress autosave (False): whether to automatically save changes to samples emitted by this iterator @@ -2178,10 +2178,9 @@ def make_label(): with contextlib.ExitStack() as exit_context: samples = self._iter_samples() - if progress: - pb = fou.ProgressBar(total=len(self)) - exit_context.enter_context(pb) - samples = pb(samples) + pb = fou.ProgressBar(total=len(self), progress=progress) + exit_context.enter_context(pb) + samples = pb(samples) if autosave: save_context = foc.SaveContext(self, batch_size=batch_size) @@ -2224,7 +2223,7 @@ def make_sample(d): def iter_groups( self, group_slices=None, - progress=False, + progress=None, autosave=False, batch_size=None, ): @@ -2265,7 +2264,7 @@ def make_label(): Args: group_slices (None): an optional subset of group slices to load - progress (False): whether to render a progress bar tracking the + progress (None): whether to render a progress bar tracking the iterator's progress autosave (False): whether to automatically save changes to samples emitted by this iterator @@ -2283,10 +2282,9 @@ def make_label(): with contextlib.ExitStack() as exit_context: groups = self._iter_groups(group_slices=group_slices) - if progress: - pb = fou.ProgressBar(total=len(self)) - exit_context.enter_context(pb) - groups = pb(groups) + pb = fou.ProgressBar(total=len(self)) + exit_context.enter_context(pb) + groups = pb(groups) if autosave: save_context = foc.SaveContext(self, batch_size=batch_size) @@ -2409,6 +2407,7 @@ def add_sample( expand_schema=True, dynamic=False, validate=True, + progress=None, ): """Adds the given sample to the dataset. @@ -2425,6 +2424,9 @@ def add_sample( document fields that are encountered validate (True): whether to validate that the fields of the sample are compliant with the dataset schema before adding it + progress (None): whether to show the progress bar of the import. + If None this uses the global setting, otherwise it overwrites + the setting for this method. Returns: the ID of the sample in the dataset @@ -2441,7 +2443,7 @@ def add_samples( dynamic=False, validate=True, num_samples=None, - progress=True, + progress=None, ): """Adds the given samples to the dataset. @@ -2463,7 +2465,9 @@ def add_samples( num_samples (None): the number of samples in ``samples``. If not provided, this is computed via ``len(samples)``, if possible. This value is optional and is used only for progress tracking - progress (True): whether to show the progress bar of the import + progress (None): whether to show the progress bar of the import. + If None this uses the global setting, otherwise it overwrites + the setting for this method. Returns: a list of IDs of the samples in the dataset @@ -2494,6 +2498,7 @@ def add_collection( include_info=True, overwrite_info=False, new_ids=False, + progress=None, ): """Adds the contents of the given collection to the dataset. @@ -2512,6 +2517,9 @@ def add_collection( information. Only applicable when ``include_info`` is True new_ids (False): whether to generate new sample/frame/group IDs. By default, the IDs of the input collection are retained + progress (None): whether to show the progress bar of the import. + If None this uses the global setting, otherwise it overwrites + the setting for this method. Returns: a list of IDs of the samples that were added to this dataset @@ -2532,6 +2540,7 @@ def add_collection( insert_new=True, include_info=include_info, overwrite_info=overwrite_info, + progress=progress, ) return self.skip(num_samples).values("id") @@ -2576,6 +2585,7 @@ def _upsert_samples( dynamic=False, validate=True, num_samples=None, + progress=None, ): if num_samples is None: try: @@ -2583,7 +2593,7 @@ def _upsert_samples( except: pass - batcher = fou.get_default_batcher(samples, progress=True) + batcher = fou.get_default_batcher(samples, progress=progress) with batcher: for batch in batcher: @@ -2808,6 +2818,7 @@ def merge_samples( include_info=True, overwrite_info=False, num_samples=None, + progress=None, ): """Merges the given samples into this dataset. @@ -2989,6 +3000,7 @@ def merge_samples( expand_schema=expand_schema, dynamic=dynamic, num_samples=num_samples, + progress=progress, ) def delete_samples(self, samples_or_ids): @@ -7604,6 +7616,7 @@ def _merge_samples_python( expand_schema=True, dynamic=False, num_samples=None, + progress=None, ): if dataset.media_type == fom.GROUP: dst = dataset.select_group_slices(_allow_mixed=True) @@ -7628,7 +7641,7 @@ def _merge_samples_python( else: id_map = {} logger.info("Indexing dataset...") - for sample in dst.iter_samples(progress=True): + for sample in dst.iter_samples(progress=progress): id_map[key_fcn(sample)] = sample._id _samples = _make_merge_samples_generator( @@ -7651,6 +7664,7 @@ def _merge_samples_python( expand_schema=expand_schema, dynamic=dynamic, num_samples=num_samples, + progress=progress, ) diff --git a/fiftyone/core/metadata.py b/fiftyone/core/metadata.py index 86737ebd7d..f64d9c7ea9 100644 --- a/fiftyone/core/metadata.py +++ b/fiftyone/core/metadata.py @@ -235,6 +235,7 @@ def compute_metadata( num_workers=None, skip_failures=True, warn_failures=False, + progress=None, ): """Populates the ``metadata`` field of all samples in the collection. @@ -250,6 +251,9 @@ def compute_metadata( error if metadata cannot be computed for a sample warn_failures (False): whether to log a warning if metadata cannot be computed for a sample + progress (None): whether to show the progress bar of the import. + If None this uses the global setting, otherwise it overwrites + the setting for this method """ num_workers = fou.recommend_thread_pool_workers(num_workers) @@ -259,10 +263,15 @@ def compute_metadata( ) if num_workers <= 1: - _compute_metadata(sample_collection, overwrite=overwrite) + _compute_metadata( + sample_collection, overwrite=overwrite, progress=progress + ) else: _compute_metadata_multi( - sample_collection, num_workers, overwrite=overwrite + sample_collection, + num_workers, + overwrite=overwrite, + progress=progress, ) if skip_failures and not warn_failures: @@ -332,7 +341,9 @@ def get_image_info(f): return width, height, len(img.getbands()) -def _compute_metadata(sample_collection, overwrite=False, batch_size=1000): +def _compute_metadata( + sample_collection, overwrite=False, batch_size=1000, progress=None +): if not overwrite: sample_collection = sample_collection.exists("metadata", False) @@ -351,7 +362,7 @@ def _compute_metadata(sample_collection, overwrite=False, batch_size=1000): values = {} try: - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: for args in pb(inputs): sample_id, metadata = _do_compute_metadata(args) values[sample_id] = metadata @@ -365,7 +376,11 @@ def _compute_metadata(sample_collection, overwrite=False, batch_size=1000): def _compute_metadata_multi( - sample_collection, num_workers, overwrite=False, batch_size=1000 + sample_collection, + num_workers, + overwrite=False, + batch_size=1000, + progress=None, ): if not overwrite: sample_collection = sample_collection.exists("metadata", False) @@ -386,7 +401,7 @@ def _compute_metadata_multi( try: with multiprocessing.dummy.Pool(processes=num_workers) as pool: - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: for sample_id, metadata in pb( pool.imap_unordered(_do_compute_metadata, inputs) ): diff --git a/fiftyone/core/utils.py b/fiftyone/core/utils.py index 5a29730c12..4fa62bf75d 100644 --- a/fiftyone/core/utils.py +++ b/fiftyone/core/utils.py @@ -942,9 +942,17 @@ def __exit__(self, *args): class ProgressBar(etau.ProgressBar): """.. autoclass:: eta.core.utils.ProgressBar""" - def __init__(self, *args, **kwargs): - if "quiet" not in kwargs: - kwargs["quiet"] = not fo.config.show_progress_bars + def __init__(self, *args, progress=None, quiet=None, **kwargs): + if quiet is not None: + # Allow overwrite with expected progress attribute + progress = not quiet + + if progress is None: + # Use global config value + progress = fo.config.show_progress_bars + + self._progress = progress + kwargs["quiet"] = not self._progress if "iters_str" not in kwargs: kwargs["iters_str"] = "samples" @@ -966,7 +974,7 @@ def __init__( self, iterable, return_views=False, - progress=False, + progress=None, total=None, ): import fiftyone.core.collections as foc diff --git a/fiftyone/zoo/datasets/__init__.py b/fiftyone/zoo/datasets/__init__.py index 0a68df2a71..eac04b7415 100644 --- a/fiftyone/zoo/datasets/__init__.py +++ b/fiftyone/zoo/datasets/__init__.py @@ -188,6 +188,7 @@ def load_zoo_dataset( persistent=False, overwrite=False, cleanup=True, + progress=None, **kwargs, ): """Loads the dataset of the given name from the FiftyOne Dataset Zoo as @@ -235,6 +236,9 @@ def load_zoo_dataset( dataset is to be downloaded cleanup (True): whether to cleanup any temporary files generated during download + progress (None): whether to show the progress bar of the import. + If None this uses the global setting, otherwise it overwrites + the setting for this method. **kwargs: optional arguments to pass to the :class:`fiftyone.utils.data.importers.DatasetImporter` constructor. If ``download_if_necessary == True``, then ``kwargs`` can also diff --git a/tests/unittests/utils_tests.py b/tests/unittests/utils_tests.py index 17a40dd270..3e91604ec5 100644 --- a/tests/unittests/utils_tests.py +++ b/tests/unittests/utils_tests.py @@ -455,6 +455,44 @@ def test_load_dataset_nonexistent(self, mock_get_db_conn): ) +class ProgressBarTests(unittest.TestCase): + def _test_correct_value(self, progress, global_progress, quiet, expected): + fo.config.show_progress_bars = global_progress + with fou.ProgressBar(list(), progress=progress, quiet=quiet) as pb: + assert pb._progress == expected + + def test_progress_None_uses_global(self): + self._test_correct_value( + progress=None, global_progress=True, quiet=None, expected=True + ) + self._test_correct_value( + progress=None, global_progress=False, quiet=None, expected=False + ) + + def test_progress_overwrites_global(self): + self._test_correct_value( + progress=True, global_progress=True, quiet=None, expected=True + ) + self._test_correct_value( + progress=True, global_progress=False, quiet=None, expected=True + ) + self._test_correct_value( + progress=False, global_progress=True, quiet=None, expected=False + ) + self._test_correct_value( + progress=False, global_progress=False, quiet=None, expected=False + ) + + def test_quiet_overwrites_all(self): + # Careful, we expect here to have progress the opposite value of quiet, as they are opposites + self._test_correct_value( + progress=True, global_progress=True, quiet=True, expected=False + ) + self._test_correct_value( + progress=False, global_progress=False, quiet=False, expected=True + ) + + if __name__ == "__main__": fo.config.show_progress_bars = False unittest.main(verbosity=2) From 74219a708a9fef54254d65aa94edbeab20f71545 Mon Sep 17 00:00:00 2001 From: Sebastian Markgraf Date: Tue, 20 Dec 2022 10:12:49 +0000 Subject: [PATCH 03/12] feature(progress): Update progress parameter where possible --- fiftyone/core/collections.py | 13 ++++++----- fiftyone/core/dataset.py | 13 ++++++----- fiftyone/core/odm/database.py | 28 +++++++++++++++++------- fiftyone/core/view.py | 22 +++++++++---------- fiftyone/utils/annotations.py | 41 +++++++++++++++++++++++++++++------ 5 files changed, 79 insertions(+), 38 deletions(-) diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py index 8eee92aab4..9bd3e7049a 100644 --- a/fiftyone/core/collections.py +++ b/fiftyone/core/collections.py @@ -826,11 +826,11 @@ def view(self): """ raise NotImplementedError("Subclass must implement view()") - def iter_samples(self, progress=False, autosave=False, batch_size=None): + def iter_samples(self, progress=None, autosave=False, batch_size=None): """Returns an iterator over the samples in the collection. Args: - progress (False): whether to render a progress bar tracking the + progress (None): whether to render a progress bar tracking the iterator's progress autosave (False): whether to automatically save changes to samples emitted by this iterator @@ -847,7 +847,7 @@ def iter_samples(self, progress=False, autosave=False, batch_size=None): def iter_groups( self, group_slices=None, - progress=False, + progress=None, autosave=False, batch_size=None, ): @@ -855,7 +855,7 @@ def iter_groups( Args: group_slices (None): an optional subset of group slices to load - progress (False): whether to render a progress bar tracking the + progress (None): whether to render a progress bar tracking the iterator's progress autosave (False): whether to automatically save changes to samples emitted by this iterator @@ -9029,6 +9029,7 @@ def to_dict( include_frames=False, frame_labels_dir=None, pretty_print=False, + progress=None, ): """Returns a JSON dictionary representation of the collection. @@ -9053,6 +9054,8 @@ def to_dict( readable format with newlines and indentations. Only applicable to datasets that contain videos when a ``frame_labels_dir`` is provided + progress (None): whether to render a progress bar tracking the + iterator's progress of the sample serialization Returns: a JSON dict @@ -9108,7 +9111,7 @@ def to_dict( # Serialize samples samples = [] - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): sd = sample.to_dict( include_frames=include_frames, include_private=include_private, diff --git a/fiftyone/core/dataset.py b/fiftyone/core/dataset.py index b3fb934c2a..e77208703f 100644 --- a/fiftyone/core/dataset.py +++ b/fiftyone/core/dataset.py @@ -2282,7 +2282,7 @@ def make_label(): with contextlib.ExitStack() as exit_context: groups = self._iter_groups(group_slices=group_slices) - pb = fou.ProgressBar(total=len(self)) + pb = fou.ProgressBar(total=len(self), progress=progress) exit_context.enter_context(pb) groups = pb(groups) @@ -2407,7 +2407,6 @@ def add_sample( expand_schema=True, dynamic=False, validate=True, - progress=None, ): """Adds the given sample to the dataset. @@ -2424,9 +2423,6 @@ def add_sample( document fields that are encountered validate (True): whether to validate that the fields of the sample are compliant with the dataset schema before adding it - progress (None): whether to show the progress bar of the import. - If None this uses the global setting, otherwise it overwrites - the setting for this method. Returns: the ID of the sample in the dataset @@ -2913,6 +2909,8 @@ def merge_samples( num_samples (None): the number of samples in ``samples``. If not provided, this is computed via ``len(samples)``, if possible. This value is optional and is used only for progress tracking + progress (None): whether to render a progress bar tracking the + progress """ if fields is not None: if etau.is_str(fields): @@ -2966,7 +2964,10 @@ def merge_samples( try: tmp.add_samples( - samples, dynamic=dynamic, num_samples=num_samples + samples, + dynamic=dynamic, + num_samples=num_samples, + progress=progress, ) self.merge_samples( diff --git a/fiftyone/core/odm/database.py b/fiftyone/core/odm/database.py index 3b24dacaac..ef0da2e705 100644 --- a/fiftyone/core/odm/database.py +++ b/fiftyone/core/odm/database.py @@ -631,6 +631,7 @@ def export_collection( key="documents", patt="{idx:06d}-{id}.json", num_docs=None, + progress=None, ): """Exports the collection to disk in JSON format. @@ -647,22 +648,31 @@ def export_collection( to the document's ID num_docs (None): the total number of documents. If omitted, this must be computable via ``len(docs)`` + progress (None): whether to render a progress bar tracking the + progress. None uses the global setting, True or False overwrite + the value for the current method. """ if num_docs is None: num_docs = len(docs) if json_dir_or_path.endswith(".json"): - _export_collection_single(docs, json_dir_or_path, key, num_docs) + _export_collection_single( + docs, json_dir_or_path, key, num_docs, progress=progress + ) else: - _export_collection_multi(docs, json_dir_or_path, patt, num_docs) + _export_collection_multi( + docs, json_dir_or_path, patt, num_docs, progress=progress + ) -def _export_collection_single(docs, json_path, key, num_docs): +def _export_collection_single(docs, json_path, key, num_docs, progress=None): etau.ensure_basedir(json_path) with open(json_path, "w") as f: f.write('{"%s": [' % key) - with fou.ProgressBar(total=num_docs, iters_str="docs") as pb: + with fou.ProgressBar( + total=num_docs, iters_str="docs", progress=progress + ) as pb: for idx, doc in pb(enumerate(docs, 1)): f.write(json_util.dumps(doc)) if idx < num_docs: @@ -671,11 +681,13 @@ def _export_collection_single(docs, json_path, key, num_docs): f.write("]}") -def _export_collection_multi(docs, json_dir, patt, num_docs): +def _export_collection_multi(docs, json_dir, patt, num_docs, progress=None): etau.ensure_dir(json_dir) json_patt = os.path.join(json_dir, patt) - with fou.ProgressBar(total=num_docs, iters_str="docs") as pb: + with fou.ProgressBar( + total=num_docs, iters_str="docs", progress=progress + ) as pb: for idx, doc in pb(enumerate(docs, 1)): json_path = json_patt.format(idx=idx, id=str(doc["_id"])) export_document(doc, json_path) @@ -735,7 +747,7 @@ def _import_collection_multi(json_dir): return docs, len(json_paths) -def insert_documents(docs, coll, ordered=False, progress=False, num_docs=None): +def insert_documents(docs, coll, ordered=False, progress=None, num_docs=None): """Inserts documents into a collection. The ``_id`` field of the input documents will be populated if it is not @@ -745,7 +757,7 @@ def insert_documents(docs, coll, ordered=False, progress=False, num_docs=None): docs: an iterable of BSON document dicts coll: a pymongo collection ordered (False): whether the documents must be inserted in order - progress (False): whether to render a progress bar tracking the + progress (None): whether to render a progress bar tracking the insertion num_docs (None): the total number of documents. Only used when ``progress=True``. If omitted, this will be computed via diff --git a/fiftyone/core/view.py b/fiftyone/core/view.py index c9da51250b..1e1a9864d2 100644 --- a/fiftyone/core/view.py +++ b/fiftyone/core/view.py @@ -431,7 +431,7 @@ def view(self): """ return copy(self) - def iter_samples(self, progress=False, autosave=False, batch_size=None): + def iter_samples(self, progress=None, autosave=False, batch_size=None): """Returns an iterator over the samples in the view. Examples:: @@ -466,7 +466,7 @@ def make_label(): sample.ground_truth.label = make_label() Args: - progress (False): whether to render a progress bar tracking the + progress (None): whether to render a progress bar tracking the iterator's progress autosave (False): whether to automatically save changes to samples emitted by this iterator @@ -480,10 +480,9 @@ def make_label(): with contextlib.ExitStack() as exit_context: samples = self._iter_samples() - if progress: - pb = fou.ProgressBar(total=len(self)) - exit_context.enter_context(pb) - samples = pb(samples) + pb = fou.ProgressBar(total=len(self), progress=progress) + exit_context.enter_context(pb) + samples = pb(samples) if autosave: save_context = foc.SaveContext(self, batch_size=batch_size) @@ -540,7 +539,7 @@ def make_sample(d): def iter_groups( self, group_slices=None, - progress=False, + progress=None, autosave=False, batch_size=None, ): @@ -582,7 +581,7 @@ def make_label(): Args: group_slices (None): an optional subset of group slices to load - progress (False): whether to render a progress bar tracking the + progress (None): whether to render a progress bar tracking the iterator's progress autosave (False): whether to automatically save changes to samples emitted by this iterator @@ -605,10 +604,9 @@ def make_label(): with contextlib.ExitStack() as exit_context: groups = self._iter_groups(group_slices=group_slices) - if progress: - pb = fou.ProgressBar(total=len(self)) - exit_context.enter_context(pb) - groups = pb(groups) + pb = fou.ProgressBar(total=len(self), progress=progress) + exit_context.enter_context(pb) + groups = pb(groups) if autosave: save_context = foc.SaveContext(self, batch_size=batch_size) diff --git a/fiftyone/utils/annotations.py b/fiftyone/utils/annotations.py index aecac59a17..f7a3bf9851 100644 --- a/fiftyone/utils/annotations.py +++ b/fiftyone/utils/annotations.py @@ -991,6 +991,7 @@ def load_annotations( dest_field=None, unexpected="prompt", cleanup=False, + progress=None, **kwargs, ): """Downloads the labels from the given annotation run from the annotation @@ -1019,6 +1020,7 @@ def load_annotations( or ``None`` if there aren't any cleanup (False): whether to delete any informtation regarding this run from the annotation backend after loading the annotations + progress (None): whether to render a progress bar tracking the progress **kwargs: keyword arguments for the run's :meth:`fiftyone.core.annotation.AnnotationMethodConfig.load_credentials` method @@ -1068,6 +1070,7 @@ def load_annotations( results, label_field, label_info=label_info, + progress=progress, ) else: _merge_labels( @@ -1079,6 +1082,7 @@ def load_annotations( label_info=label_info, global_attrs=global_attrs, class_attrs=class_attrs, + progress=progress, ) else: # Unexpected labels @@ -1107,10 +1111,21 @@ def load_annotations( ) if anno_type == "scalar": - _merge_scalars(dataset, annos, results, new_field) + _merge_scalars( + dataset, + annos, + results, + new_field, + progress=progress, + ) else: _merge_labels( - dataset, annos, results, new_field, anno_type + dataset, + annos, + results, + new_field, + anno_type, + progress=progress, ) else: if label_field: @@ -1256,7 +1271,9 @@ def _prompt_field(dataset, label_type, label_field, label_schema): return new_field -def _merge_scalars(dataset, anno_dict, results, label_field, label_info=None): +def _merge_scalars( + dataset, anno_dict, results, label_field, label_info=None, progress=None +): if label_info is None: label_info = {} @@ -1290,7 +1307,7 @@ def _merge_scalars(dataset, anno_dict, results, label_field, label_info=None): num_deletions = 0 logger.info("Loading scalars for field '%s'...", label_field) - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): sample_annos = anno_dict.get(sample.id, None) if is_frame_field: @@ -1355,6 +1372,7 @@ def _merge_labels( label_info=None, global_attrs=None, class_attrs=None, + progress=None, ): if label_info is None: label_info = {} @@ -1472,7 +1490,7 @@ def _merge_labels( # Add/merge labels from the annotation task sample_ids = list(anno_dict.keys()) view = view.select(sample_ids).select_fields(label_field) - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): sample_id = sample.id sample_annos = anno_dict[sample_id] @@ -2294,7 +2312,13 @@ def __init__(self, d): def draw_labeled_images( - samples, output_dir, rel_dir=None, label_fields=None, config=None, **kwargs + samples, + output_dir, + rel_dir=None, + label_fields=None, + config=None, + progress=None, + **kwargs, ): """Renders annotated versions of the images in the collection with the specified label data overlaid to the given directory. @@ -2319,6 +2343,9 @@ def draw_labeled_images( If omitted, all compatiable fields are rendered config (None): an optional :class:`DrawConfig` configuring how to draw the labels + progress (None): whether to render a progress bar tracking the + progress. None uses the global setting, True or False overwrite + the value for the current method. **kwargs: optional keyword arguments specifying parameters of the default :class:`DrawConfig` to override @@ -2335,7 +2362,7 @@ def draw_labeled_images( output_ext = fo.config.default_image_ext outpaths = [] - for sample in samples.iter_samples(progress=True): + for sample in samples.iter_samples(progress=progress): outpath = filename_maker.get_output_path( sample.filepath, output_ext=output_ext ) From d4adb2c30d533624f4cf489439aec82e9f12dd84 Mon Sep 17 00:00:00 2001 From: brimoor Date: Sat, 22 Jul 2023 14:35:10 -0400 Subject: [PATCH 04/12] more progress flags --- fiftyone/core/collections.py | 42 ++++- fiftyone/core/dataset.py | 228 +++++++++++++++++++------- fiftyone/core/metadata.py | 4 +- fiftyone/core/models.py | 95 ++++++++--- fiftyone/core/odm/database.py | 7 +- fiftyone/core/utils.py | 26 ++- fiftyone/core/view.py | 33 ++-- fiftyone/utils/annotations.py | 6 +- fiftyone/utils/coco.py | 23 --- fiftyone/utils/csv.py | 4 +- fiftyone/utils/cvat.py | 10 +- fiftyone/utils/data/exporters.py | 83 ++++++---- fiftyone/utils/data/importers.py | 45 ++--- fiftyone/utils/data/parsers.py | 49 +++--- fiftyone/utils/eval/activitynet.py | 11 +- fiftyone/utils/eval/classification.py | 17 +- fiftyone/utils/eval/coco.py | 17 +- fiftyone/utils/eval/detection.py | 20 ++- fiftyone/utils/eval/openimages.py | 9 +- fiftyone/utils/eval/regression.py | 13 +- fiftyone/utils/eval/segmentation.py | 26 ++- fiftyone/utils/geojson.py | 9 +- fiftyone/utils/image.py | 15 +- fiftyone/utils/iou.py | 14 +- fiftyone/utils/labelbox.py | 27 +-- fiftyone/utils/labels.py | 52 ++++-- fiftyone/utils/scale.py | 8 +- fiftyone/utils/utils3d.py | 4 +- fiftyone/utils/video.py | 12 +- tests/unittests/utils_tests.py | 4 +- 30 files changed, 618 insertions(+), 295 deletions(-) diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py index 9bd3e7049a..9c601d2fe9 100644 --- a/fiftyone/core/collections.py +++ b/fiftyone/core/collections.py @@ -826,12 +826,11 @@ def view(self): """ raise NotImplementedError("Subclass must implement view()") - def iter_samples(self, progress=None, autosave=False, batch_size=None): + def iter_samples(self, progress=False, autosave=False, batch_size=None): """Returns an iterator over the samples in the collection. Args: - progress (None): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -847,7 +846,7 @@ def iter_samples(self, progress=None, autosave=False, batch_size=None): def iter_groups( self, group_slices=None, - progress=None, + progress=False, autosave=False, batch_size=None, ): @@ -855,8 +854,7 @@ def iter_groups( Args: group_slices (None): an optional subset of group slices to load - progress (None): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -2735,6 +2733,7 @@ def compute_metadata( num_workers=None, skip_failures=True, warn_failures=False, + progress=None, ): """Populates the ``metadata`` field of all samples in the collection. @@ -2748,6 +2747,7 @@ def compute_metadata( raising an error if metadata cannot be computed for a sample warn_failures (False): whether to log a warning if metadata cannot be computed for a sample + progress (None): whether to render a progress bar """ fomt.compute_metadata( self, @@ -2755,6 +2755,7 @@ def compute_metadata( num_workers=num_workers, skip_failures=skip_failures, warn_failures=warn_failures, + progress=progress, ) def apply_model( @@ -2768,6 +2769,7 @@ def apply_model( skip_failures=True, output_dir=None, rel_dir=None, + progress=None, **kwargs, ): """Applies the :class:`FiftyOne model ` or @@ -2816,6 +2818,7 @@ def apply_model( subdirectories in ``output_dir`` that match the shape of the input paths. The path is converted to an absolute path (if necessary) via :func:`fiftyone.core.storage.normalize_path` + progress (None): whether to render a progress bar **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation """ @@ -2830,6 +2833,7 @@ def apply_model( skip_failures=skip_failures, output_dir=output_dir, rel_dir=rel_dir, + progress=progress, **kwargs, ) @@ -2840,6 +2844,7 @@ def compute_embeddings( batch_size=None, num_workers=None, skip_failures=True, + progress=None, **kwargs, ): """Computes embeddings for the samples in the collection using the @@ -2879,6 +2884,7 @@ def compute_embeddings( raising an error if embeddings cannot be generated for a sample. Only applicable to :class:`fiftyone.core.models.Model` instances + progress (None): whether to render a progress bar **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation @@ -2908,6 +2914,7 @@ def compute_embeddings( batch_size=batch_size, num_workers=num_workers, skip_failures=skip_failures, + progress=progress, **kwargs, ) @@ -2922,6 +2929,7 @@ def compute_patch_embeddings( batch_size=None, num_workers=None, skip_failures=True, + progress=None, ): """Computes embeddings for the image patches defined by ``patches_field`` of the samples in the collection using the given @@ -2973,6 +2981,7 @@ def compute_patch_embeddings( applicable for Torch-based models skip_failures (True): whether to gracefully continue without raising an error if embeddings cannot be generated for a sample + progress (None): whether to render a progress bar Returns: one of the following: @@ -3003,6 +3012,7 @@ def compute_patch_embeddings( alpha=alpha, handle_missing=handle_missing, skip_failures=skip_failures, + progress=progress, ) def evaluate_regressions( @@ -3012,6 +3022,7 @@ def evaluate_regressions( eval_key=None, missing=None, method=None, + progress=None, **kwargs, ): """Evaluates the regression predictions in this collection with respect @@ -3050,6 +3061,7 @@ def evaluate_regressions( The supported values are ``fo.evaluation_config.regression_backends.keys()`` and the default is ``fo.evaluation_config.regression_default_backend`` + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.regression.RegressionEvaluationConfig` being used @@ -3064,6 +3076,7 @@ def evaluate_regressions( eval_key=eval_key, missing=missing, method=method, + progress=progress, **kwargs, ) @@ -3075,6 +3088,7 @@ def evaluate_classifications( classes=None, missing=None, method=None, + progress=None, **kwargs, ): """Evaluates the classification predictions in this collection with @@ -3122,6 +3136,7 @@ def evaluate_classifications( The supported values are ``fo.evaluation_config.classification_backends.keys()`` and the default is ``fo.evaluation_config.classification_default_backend`` + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.classification.ClassificationEvaluationConfig` being used @@ -3137,6 +3152,7 @@ def evaluate_classifications( classes=classes, missing=missing, method=method, + progress=progress, **kwargs, ) @@ -3153,6 +3169,7 @@ def evaluate_detections( use_boxes=False, classwise=True, dynamic=True, + progress=None, **kwargs, ): """Evaluates the specified predicted detections in this collection with @@ -3245,6 +3262,7 @@ def evaluate_detections( label (True) or allow matches between classes (False) dynamic (True): whether to declare the dynamic object-level attributes that are populated on the dataset's schema + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.detection.DetectionEvaluationConfig` being used @@ -3265,6 +3283,7 @@ def evaluate_detections( use_boxes=use_boxes, classwise=classwise, dynamic=dynamic, + progress=progress, **kwargs, ) @@ -3275,6 +3294,7 @@ def evaluate_segmentations( eval_key=None, mask_targets=None, method=None, + progress=None, **kwargs, ): """Evaluates the specified semantic segmentation masks in this @@ -3328,6 +3348,7 @@ class for the purposes of computing evaluation metrics like The supported values are ``fo.evaluation_config.segmentation_backends.keys()`` and the default is ``fo.evaluation_config.segmentation_default_backend`` + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.segmentation.SegmentationEvaluationConfig` being used @@ -3342,6 +3363,7 @@ class for the purposes of computing evaluation metrics like eval_key=eval_key, mask_targets=mask_targets, method=method, + progress=progress, **kwargs, ) @@ -8234,6 +8256,7 @@ def export( label_field=None, frame_labels_field=None, overwrite=False, + progress=None, **kwargs, ): """Exports the samples in the collection to disk. @@ -8407,6 +8430,7 @@ def export( overwrite (False): whether to delete existing directories before performing the export (True) or to merge the export with existing files and directories (False) + progress (None): whether to render a progress bar **kwargs: optional keyword arguments to pass to the dataset exporter's constructor. If you are exporting image patches, this can also contain keyword arguments for @@ -8432,6 +8456,7 @@ def export( label_field=label_field, frame_labels_field=frame_labels_field, overwrite=overwrite, + progress=progress, **kwargs, ) @@ -9054,8 +9079,7 @@ def to_dict( readable format with newlines and indentations. Only applicable to datasets that contain videos when a ``frame_labels_dir`` is provided - progress (None): whether to render a progress bar tracking the - iterator's progress of the sample serialization + progress (None): whether to render a progress bar Returns: a JSON dict @@ -10926,6 +10950,7 @@ def _export( label_field=None, frame_labels_field=None, overwrite=False, + progress=None, **kwargs, ): if dataset_type is None and dataset_exporter is None: @@ -10997,6 +11022,7 @@ def _export( dataset_exporter=dataset_exporter, label_field=label_field, frame_labels_field=frame_labels_field, + progress=progress, **kwargs, ) diff --git a/fiftyone/core/dataset.py b/fiftyone/core/dataset.py index e77208703f..113ce26ca5 100644 --- a/fiftyone/core/dataset.py +++ b/fiftyone/core/dataset.py @@ -2130,7 +2130,7 @@ def delete_group_slice(self, name): self.save() - def iter_samples(self, progress=None, autosave=False, batch_size=None): + def iter_samples(self, progress=False, autosave=False, batch_size=None): """Returns an iterator over the samples in the dataset. Examples:: @@ -2164,8 +2164,7 @@ def make_label(): sample.ground_truth.label = make_label() Args: - progress (None): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -2175,10 +2174,13 @@ def make_label(): Returns: an iterator over :class:`fiftyone.core.sample.Sample` instances """ + if progress is None: + progress = False + with contextlib.ExitStack() as exit_context: samples = self._iter_samples() - pb = fou.ProgressBar(total=len(self), progress=progress) + pb = fou.ProgressBar(total=self, progress=progress) exit_context.enter_context(pb) samples = pb(samples) @@ -2223,7 +2225,7 @@ def make_sample(d): def iter_groups( self, group_slices=None, - progress=None, + progress=False, autosave=False, batch_size=None, ): @@ -2264,8 +2266,7 @@ def make_label(): Args: group_slices (None): an optional subset of group slices to load - progress (None): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -2279,10 +2280,13 @@ def make_label(): if self.media_type != fom.GROUP: raise ValueError("%s does not contain groups" % type(self)) + if progress is None: + progress = False + with contextlib.ExitStack() as exit_context: groups = self._iter_groups(group_slices=group_slices) - pb = fou.ProgressBar(total=len(self), progress=progress) + pb = fou.ProgressBar(total=self, progress=progress) exit_context.enter_context(pb) groups = pb(groups) @@ -2438,8 +2442,8 @@ def add_samples( expand_schema=True, dynamic=False, validate=True, - num_samples=None, progress=None, + num_samples=None, ): """Adds the given samples to the dataset. @@ -2458,21 +2462,16 @@ def add_samples( document fields that are encountered validate (True): whether to validate that the fields of each sample are compliant with the dataset schema before adding it + progress (None): whether to render a progress bar num_samples (None): the number of samples in ``samples``. If not - provided, this is computed via ``len(samples)``, if possible. - This value is optional and is used only for progress tracking - progress (None): whether to show the progress bar of the import. - If None this uses the global setting, otherwise it overwrites - the setting for this method. + provided, this is computed (if possible) via ``len(samples)`` + if needed for progress tracking Returns: a list of IDs of the samples in the dataset """ if num_samples is None: - try: - num_samples = len(samples) - except: - pass + num_samples = samples batcher = fou.get_default_batcher( samples, progress=progress, total=num_samples @@ -2513,9 +2512,7 @@ def add_collection( information. Only applicable when ``include_info`` is True new_ids (False): whether to generate new sample/frame/group IDs. By default, the IDs of the input collection are retained - progress (None): whether to show the progress bar of the import. - If None this uses the global setting, otherwise it overwrites - the setting for this method. + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to this dataset @@ -2580,14 +2577,11 @@ def _upsert_samples( expand_schema=True, dynamic=False, validate=True, - num_samples=None, progress=None, + num_samples=None, ): if num_samples is None: - try: - num_samples = len(samples) - except: - pass + num_samples = samples batcher = fou.get_default_batcher(samples, progress=progress) @@ -2813,8 +2807,8 @@ def merge_samples( dynamic=False, include_info=True, overwrite_info=False, - num_samples=None, progress=None, + num_samples=None, ): """Merges the given samples into this dataset. @@ -2906,11 +2900,10 @@ def merge_samples( information. Only applicable when ``samples`` is a :class:`fiftyone.core.collections.SampleCollection` and ``include_info`` is True + progress (None): whether to render a progress bar num_samples (None): the number of samples in ``samples``. If not - provided, this is computed via ``len(samples)``, if possible. - This value is optional and is used only for progress tracking - progress (None): whether to render a progress bar tracking the - progress + provided, this is computed (if possibleE) via ``len(samples)`` + if needed for progress tracking """ if fields is not None: if etau.is_str(fields): @@ -2966,8 +2959,8 @@ def merge_samples( tmp.add_samples( samples, dynamic=dynamic, - num_samples=num_samples, progress=progress, + num_samples=num_samples, ) self.merge_samples( @@ -3000,8 +2993,8 @@ def merge_samples( overwrite=overwrite, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, progress=progress, + num_samples=num_samples, ) def delete_samples(self, samples_or_ids): @@ -3977,6 +3970,7 @@ def add_dir( expand_schema=True, dynamic=False, add_info=True, + progress=None, **kwargs, ): """Adds the contents of the given directory to the dataset. @@ -4065,6 +4059,7 @@ def add_dir( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset's ``info`` + progress (None): whether to render a progress bar **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4088,6 +4083,7 @@ def add_dir( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, ) def merge_dir( @@ -4109,6 +4105,7 @@ def merge_dir( expand_schema=True, dynamic=False, add_info=True, + progress=None, **kwargs, ): """Merges the contents of the given directory into the dataset. @@ -4266,6 +4263,7 @@ def merge_dir( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset + progress (None): whether to render a progress bar **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4294,6 +4292,7 @@ def merge_dir( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, ) def add_archive( @@ -4308,6 +4307,7 @@ def add_archive( dynamic=False, add_info=True, cleanup=True, + progress=None, **kwargs, ): """Adds the contents of the given archive to the dataset. @@ -4392,6 +4392,7 @@ def add_archive( add_info (True): whether to add dataset info from the importer (if any) to the dataset's ``info`` cleanup (True): whether to delete the archive after extracting it + progress (None): whether to render a progress bar **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4410,6 +4411,7 @@ def add_archive( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, **kwargs, ) @@ -4433,6 +4435,7 @@ def merge_archive( dynamic=False, add_info=True, cleanup=True, + progress=None, **kwargs, ): """Merges the contents of the given archive into the dataset. @@ -4586,6 +4589,7 @@ def merge_archive( add_info (True): whether to add dataset info from the importer (if any) to the dataset cleanup (True): whether to delete the archive after extracting it + progress (None): whether to render a progress bar **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4609,6 +4613,7 @@ def merge_archive( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, **kwargs, ) @@ -4620,6 +4625,7 @@ def add_importer( expand_schema=True, dynamic=False, add_info=True, + progress=None, ): """Adds the samples from the given :class:`fiftyone.utils.data.importers.DatasetImporter` to the dataset. @@ -4653,6 +4659,7 @@ def add_importer( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset's ``info`` + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -4665,6 +4672,7 @@ def add_importer( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, ) def merge_importer( @@ -4683,6 +4691,7 @@ def merge_importer( expand_schema=True, dynamic=False, add_info=True, + progress=None, ): """Merges the samples from the given :class:`fiftyone.utils.data.importers.DatasetImporter` into the @@ -4786,6 +4795,7 @@ def merge_importer( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset + progress (None): whether to render a progress bar """ return foud.merge_samples( self, @@ -4803,9 +4813,16 @@ def merge_importer( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, ) - def add_images(self, paths_or_samples, sample_parser=None, tags=None): + def add_images( + self, + paths_or_samples, + sample_parser=None, + tags=None, + progress=None, + ): """Adds the given images to the dataset. This operation does not read the images. @@ -4824,6 +4841,7 @@ def add_images(self, paths_or_samples, sample_parser=None, tags=None): instance to use to parse the samples tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -4832,7 +4850,11 @@ def add_images(self, paths_or_samples, sample_parser=None, tags=None): sample_parser = foud.ImageSampleParser() return foud.add_images( - self, paths_or_samples, sample_parser, tags=tags + self, + paths_or_samples, + sample_parser, + tags=tags, + progress=progress, ) def add_labeled_images( @@ -4843,6 +4865,7 @@ def add_labeled_images( tags=None, expand_schema=True, dynamic=False, + progress=None, ): """Adds the given labeled images to the dataset. @@ -4875,6 +4898,7 @@ def add_labeled_images( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -4887,9 +4911,16 @@ def add_labeled_images( tags=tags, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, ) - def add_images_dir(self, images_dir, tags=None, recursive=True): + def add_images_dir( + self, + images_dir, + tags=None, + recursive=True, + progress=None, + ): """Adds the given directory of images to the dataset. See :class:`fiftyone.types.ImageDirectory` for format details. In @@ -4902,15 +4933,18 @@ def add_images_dir(self, images_dir, tags=None, recursive=True): tags (None): an optional tag or iterable of tags to attach to each sample recursive (True): whether to recursively traverse subdirectories + progress (None): whether to render a progress bar Returns: a list of IDs of the samples in the dataset """ image_paths = foud.parse_images_dir(images_dir, recursive=recursive) sample_parser = foud.ImageSampleParser() - return self.add_images(image_paths, sample_parser, tags=tags) + return self.add_images( + image_paths, sample_parser, tags=tags, progress=progress + ) - def add_images_patt(self, images_patt, tags=None): + def add_images_patt(self, images_patt, tags=None, progress=None): """Adds the given glob pattern of images to the dataset. This operation does not read the images. @@ -4920,13 +4954,16 @@ def add_images_patt(self, images_patt, tags=None): ``/path/to/images/*.jpg`` tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a list of IDs of the samples in the dataset """ image_paths = etau.get_glob_matches(images_patt) sample_parser = foud.ImageSampleParser() - return self.add_images(image_paths, sample_parser, tags=tags) + return self.add_images( + image_paths, sample_parser, tags=tags, progress=progress + ) def ingest_images( self, @@ -4935,6 +4972,7 @@ def ingest_images( tags=None, dataset_dir=None, image_format=None, + progress=None, ): """Ingests the given iterable of images into the dataset. @@ -4975,7 +5013,9 @@ def ingest_images( image_format=image_format, ) - return self.add_importer(dataset_ingestor, tags=tags) + return self.add_importer( + dataset_ingestor, tags=tags, progress=progress + ) def ingest_labeled_images( self, @@ -4987,6 +5027,7 @@ def ingest_labeled_images( dynamic=False, dataset_dir=None, image_format=None, + progress=None, ): """Ingests the given iterable of labeled image samples into the dataset. @@ -5022,6 +5063,7 @@ def ingest_labeled_images( written. By default, :func:`get_default_dataset_dir` is used image_format (None): the image format to use to write the images to disk. By default, ``fiftyone.config.default_image_ext`` is used + progress (None): whether to render a progress bar Returns: a list of IDs of the samples in the dataset @@ -5042,9 +5084,16 @@ def ingest_labeled_images( tags=tags, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, ) - def add_videos(self, paths_or_samples, sample_parser=None, tags=None): + def add_videos( + self, + paths_or_samples, + sample_parser=None, + tags=None, + progress=None, + ): """Adds the given videos to the dataset. This operation does not read the videos. @@ -5063,6 +5112,7 @@ def add_videos(self, paths_or_samples, sample_parser=None, tags=None): instance to use to parse the samples tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -5071,7 +5121,7 @@ def add_videos(self, paths_or_samples, sample_parser=None, tags=None): sample_parser = foud.VideoSampleParser() return foud.add_videos( - self, paths_or_samples, sample_parser, tags=tags + self, paths_or_samples, sample_parser, tags=tags, progress=progress ) def add_labeled_videos( @@ -5082,6 +5132,7 @@ def add_labeled_videos( tags=None, expand_schema=True, dynamic=False, + progress=None, ): """Adds the given labeled videos to the dataset. @@ -5116,6 +5167,7 @@ def add_labeled_videos( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -5128,9 +5180,12 @@ def add_labeled_videos( tags=tags, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, ) - def add_videos_dir(self, videos_dir, tags=None, recursive=True): + def add_videos_dir( + self, videos_dir, tags=None, recursive=True, progress=None + ): """Adds the given directory of videos to the dataset. See :class:`fiftyone.types.VideoDirectory` for format details. In @@ -5143,15 +5198,18 @@ def add_videos_dir(self, videos_dir, tags=None, recursive=True): tags (None): an optional tag or iterable of tags to attach to each sample recursive (True): whether to recursively traverse subdirectories + progress (None): whether to render a progress bar Returns: a list of IDs of the samples in the dataset """ video_paths = foud.parse_videos_dir(videos_dir, recursive=recursive) sample_parser = foud.VideoSampleParser() - return self.add_videos(video_paths, sample_parser, tags=tags) + return self.add_videos( + video_paths, sample_parser, tags=tags, progress=progress + ) - def add_videos_patt(self, videos_patt, tags=None): + def add_videos_patt(self, videos_patt, tags=None, progress=None): """Adds the given glob pattern of videos to the dataset. This operation does not read/decode the videos. @@ -5161,13 +5219,16 @@ def add_videos_patt(self, videos_patt, tags=None): ``/path/to/videos/*.mp4`` tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a list of IDs of the samples in the dataset """ video_paths = etau.get_glob_matches(videos_patt) sample_parser = foud.VideoSampleParser() - return self.add_videos(video_paths, sample_parser, tags=tags) + return self.add_videos( + video_paths, sample_parser, tags=tags, progress=progress + ) def ingest_videos( self, @@ -5175,6 +5236,7 @@ def ingest_videos( sample_parser=None, tags=None, dataset_dir=None, + progress=None, ): """Ingests the given iterable of videos into the dataset. @@ -5196,6 +5258,7 @@ def ingest_videos( sample dataset_dir (None): the directory in which the videos will be written. By default, :func:`get_default_dataset_dir` is used + progress (None): whether to render a progress bar Returns: a list of IDs of the samples in the dataset @@ -5210,7 +5273,9 @@ def ingest_videos( dataset_dir, paths_or_samples, sample_parser ) - return self.add_importer(dataset_ingestor, tags=tags) + return self.add_importer( + dataset_ingestor, tags=tags, progress=progress + ) def ingest_labeled_videos( self, @@ -5220,6 +5285,7 @@ def ingest_labeled_videos( expand_schema=True, dynamic=False, dataset_dir=None, + progress=None, ): """Ingests the given iterable of labeled video samples into the dataset. @@ -5244,6 +5310,7 @@ def ingest_labeled_videos( document fields that are encountered dataset_dir (None): the directory in which the videos will be written. By default, :func:`get_default_dataset_dir` is used + progress (None): whether to render a progress bar Returns: a list of IDs of the samples in the dataset @@ -5260,6 +5327,7 @@ def ingest_labeled_videos( tags=tags, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, ) @classmethod @@ -5275,6 +5343,7 @@ def from_dir( label_field=None, tags=None, dynamic=False, + progress=None, **kwargs, ): """Creates a :class:`Dataset` from the contents of the given directory. @@ -5363,6 +5432,7 @@ def from_dir( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -5379,6 +5449,7 @@ def from_dir( label_field=label_field, tags=tags, dynamic=dynamic, + progress=progress, **kwargs, ) return dataset @@ -5397,6 +5468,7 @@ def from_archive( tags=None, dynamic=False, cleanup=True, + progress=None, **kwargs, ): """Creates a :class:`Dataset` from the contents of the given archive. @@ -5482,6 +5554,7 @@ def from_archive( dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered cleanup (True): whether to delete the archive after extracting it + progress (None): whether to render a progress bar **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -5499,6 +5572,7 @@ def from_archive( tags=tags, dynamic=dynamic, cleanup=cleanup, + progress=progress, **kwargs, ) return dataset @@ -5513,6 +5587,7 @@ def from_importer( label_field=None, tags=None, dynamic=False, + progress=None, ): """Creates a :class:`Dataset` by importing the samples in the given :class:`fiftyone.utils.data.importers.DatasetImporter`. @@ -5548,6 +5623,7 @@ def from_importer( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar Returns: a :class:`Dataset` @@ -5558,6 +5634,7 @@ def from_importer( label_field=label_field, tags=tags, dynamic=dynamic, + progress=progress, ) return dataset @@ -5570,6 +5647,7 @@ def from_images( persistent=False, overwrite=False, tags=None, + progress=None, ): """Creates a :class:`Dataset` from the given images. @@ -5596,13 +5674,17 @@ def from_images( the same name tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) dataset.add_images( - paths_or_samples, sample_parser=sample_parser, tags=tags + paths_or_samples, + sample_parser=sample_parser, + tags=tags, + progress=progress, ) return dataset @@ -5617,6 +5699,7 @@ def from_labeled_images( label_field=None, tags=None, dynamic=False, + progress=None, ): """Creates a :class:`Dataset` from the given labeled images. @@ -5652,6 +5735,7 @@ def from_labeled_images( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar Returns: a :class:`Dataset` @@ -5663,6 +5747,7 @@ def from_labeled_images( label_field=label_field, tags=tags, dynamic=dynamic, + progress=progress, ) return dataset @@ -5675,6 +5760,7 @@ def from_images_dir( overwrite=False, tags=None, recursive=True, + progress=None, ): """Creates a :class:`Dataset` from the given directory of images. @@ -5691,12 +5777,15 @@ def from_images_dir( tags (None): an optional tag or iterable of tags to attach to each sample recursive (True): whether to recursively traverse subdirectories + progress (None): whether to render a progress bar Returns: a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) - dataset.add_images_dir(images_dir, tags=tags, recursive=recursive) + dataset.add_images_dir( + images_dir, tags=tags, recursive=recursive, progress=progress + ) return dataset @classmethod @@ -5707,6 +5796,7 @@ def from_images_patt( persistent=False, overwrite=False, tags=None, + progress=None, ): """Creates a :class:`Dataset` from the given glob pattern of images. @@ -5723,12 +5813,13 @@ def from_images_patt( the same name tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) - dataset.add_images_patt(images_patt, tags=tags) + dataset.add_images_patt(images_patt, tags=tags, progress=progress) return dataset @classmethod @@ -5740,6 +5831,7 @@ def from_videos( persistent=False, overwrite=False, tags=None, + progress=None, ): """Creates a :class:`Dataset` from the given videos. @@ -5766,13 +5858,17 @@ def from_videos( the same name tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) dataset.add_videos( - paths_or_samples, sample_parser=sample_parser, tags=tags + paths_or_samples, + sample_parser=sample_parser, + tags=tags, + progress=progress, ) return dataset @@ -5787,6 +5883,7 @@ def from_labeled_videos( label_field=None, tags=None, dynamic=False, + progress=None, ): """Creates a :class:`Dataset` from the given labeled videos. @@ -5823,6 +5920,7 @@ def from_labeled_videos( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar Returns: a :class:`Dataset` @@ -5834,6 +5932,7 @@ def from_labeled_videos( label_field=label_field, tags=tags, dynamic=dynamic, + progress=progress, ) return dataset @@ -5846,6 +5945,7 @@ def from_videos_dir( overwrite=False, tags=None, recursive=True, + progress=None, ): """Creates a :class:`Dataset` from the given directory of videos. @@ -5867,7 +5967,9 @@ def from_videos_dir( a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) - dataset.add_videos_dir(videos_dir, tags=tags, recursive=recursive) + dataset.add_videos_dir( + videos_dir, tags=tags, recursive=recursive, progress=progress + ) return dataset @classmethod @@ -5878,6 +5980,7 @@ def from_videos_patt( persistent=False, overwrite=False, tags=None, + progress=None, ): """Creates a :class:`Dataset` from the given glob pattern of videos. @@ -5899,7 +6002,7 @@ def from_videos_patt( a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) - dataset.add_videos_patt(videos_patt, tags=tags) + dataset.add_videos_patt(videos_patt, tags=tags, progress=progress) return dataset @classmethod @@ -5911,6 +6014,7 @@ def from_dict( overwrite=False, rel_dir=None, frame_labels_dir=None, + progress=None, ): """Loads a :class:`Dataset` from a JSON dictionary generated by :meth:`fiftyone.core.collections.SampleCollection.to_dict`. @@ -5935,6 +6039,7 @@ def from_dict( is assumed that the frame labels are included directly in the provided JSON dict. Only applicable to datasets that contain videos + progress (None): whether to render a progress bar Returns: a :class:`Dataset` @@ -6014,7 +6119,10 @@ def parse_sample(sd): _samples = map(parse_sample, samples) dataset.add_samples( - _samples, expand_schema=False, num_samples=len(samples) + _samples, + expand_schema=False, + progress=progress, + num_samples=samples, ) return dataset @@ -6028,6 +6136,7 @@ def from_json( overwrite=False, rel_dir=None, frame_labels_dir=None, + progress=None, ): """Loads a :class:`Dataset` from JSON generated by :func:`fiftyone.core.collections.SampleCollection.write_json` or @@ -6048,6 +6157,7 @@ def from_json( of each sample, if the filepath is not absolute (begins with a path separator). The path is converted to an absolute path (if necessary) via :func:`fiftyone.core.storage.normalize_path` + progress (None): whether to render a progress bar Returns: a :class:`Dataset` @@ -6060,6 +6170,7 @@ def from_json( overwrite=overwrite, rel_dir=rel_dir, frame_labels_dir=frame_labels_dir, + progress=progress, ) def _add_view_stage(self, stage): @@ -7616,8 +7727,8 @@ def _merge_samples_python( overwrite=True, expand_schema=True, dynamic=False, - num_samples=None, progress=None, + num_samples=None, ): if dataset.media_type == fom.GROUP: dst = dataset.select_group_slices(_allow_mixed=True) @@ -7631,10 +7742,7 @@ def _merge_samples_python( samples = samples.select_group_slices(_allow_mixed=True) if num_samples is None: - try: - num_samples = len(samples) - except: - pass + num_samples = samples if key_fcn is None: id_map = {k: v for k, v in zip(*dst.values([key_field, "_id"]))} @@ -7664,8 +7772,8 @@ def _merge_samples_python( _samples, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, progress=progress, + num_samples=num_samples, ) diff --git a/fiftyone/core/metadata.py b/fiftyone/core/metadata.py index f64d9c7ea9..bbb714bff1 100644 --- a/fiftyone/core/metadata.py +++ b/fiftyone/core/metadata.py @@ -251,9 +251,7 @@ def compute_metadata( error if metadata cannot be computed for a sample warn_failures (False): whether to log a warning if metadata cannot be computed for a sample - progress (None): whether to show the progress bar of the import. - If None this uses the global setting, otherwise it overwrites - the setting for this method + progress (None): whether to render a progress bar """ num_workers = fou.recommend_thread_pool_workers(num_workers) diff --git a/fiftyone/core/models.py b/fiftyone/core/models.py index 1d99f30954..525212eb0d 100644 --- a/fiftyone/core/models.py +++ b/fiftyone/core/models.py @@ -56,6 +56,7 @@ def apply_model( skip_failures=True, output_dir=None, rel_dir=None, + progress=None, **kwargs, ): """Applies the :class:`FiftyOne model ` or @@ -99,6 +100,7 @@ def apply_model( ``output_dir`` that match the shape of the input paths. The path is converted to an absolute path (if necessary) via :func:`fiftyone.core.storage.normalize_path` + progress (None): whether to render a progress bar **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation """ @@ -210,6 +212,7 @@ def apply_model( confidence_thresh, skip_failures, filename_maker, + progress, ) batch_size = _parse_batch_size(batch_size, model, use_data_loader) @@ -226,6 +229,7 @@ def apply_model( batch_size, skip_failures, filename_maker, + progress, ) return _apply_image_model_to_frames_single( @@ -235,6 +239,7 @@ def apply_model( confidence_thresh, skip_failures, filename_maker, + progress, ) if use_data_loader: @@ -247,6 +252,7 @@ def apply_model( num_workers, skip_failures, filename_maker, + progress, ) if batch_size is not None: @@ -258,6 +264,7 @@ def apply_model( batch_size, skip_failures, filename_maker, + progress, ) return _apply_image_model_single( @@ -267,6 +274,7 @@ def apply_model( confidence_thresh, skip_failures, filename_maker, + progress, ) @@ -288,10 +296,11 @@ def _apply_image_model_single( confidence_thresh, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): try: img = foui.read(sample.filepath) @@ -325,11 +334,12 @@ def _apply_image_model_batch( batch_size, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) samples_loader = fou.iter_batches(samples, batch_size) - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample_batch in samples_loader: try: imgs = [foui.read(sample.filepath) for sample in sample_batch] @@ -375,6 +385,7 @@ def _apply_image_model_data_loader( num_workers, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) samples_loader = fou.iter_batches(samples, batch_size) @@ -382,7 +393,7 @@ def _apply_image_model_data_loader( samples, model, batch_size, num_workers, skip_failures ) - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample_batch, imgs in zip(samples_loader, data_loader): try: if isinstance(imgs, Exception): @@ -427,12 +438,13 @@ def _apply_image_model_to_frames_single( confidence_thresh, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) frame_counts, total_frame_count = _get_frame_counts(samples) is_clips = samples._dataset._is_clips - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): if is_clips: frames = etaf.FrameRange(*sample.support) @@ -482,12 +494,13 @@ def _apply_image_model_to_frames_batch( batch_size, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) frame_counts, total_frame_count = _get_frame_counts(samples) is_clips = samples._dataset._is_clips - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): if is_clips: frames = etaf.FrameRange(*sample.support) @@ -542,11 +555,12 @@ def _apply_video_model( confidence_thresh, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) is_clips = samples._dataset._is_clips - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): if is_clips: frames = etaf.FrameRange(*sample.support) @@ -713,6 +727,7 @@ def compute_embeddings( batch_size=None, num_workers=None, skip_failures=True, + progress=None, **kwargs, ): """Computes embeddings for the samples in the collection using the given @@ -749,6 +764,7 @@ def compute_embeddings( skip_failures (True): whether to gracefully continue without raising an error if embeddings cannot be generated for a sample. Only applicable to :class:`Model` instances + progress (None): whether to render a progress bar **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation @@ -851,7 +867,7 @@ def compute_embeddings( if samples.media_type == fom.VIDEO and model.media_type == "video": return _compute_video_embeddings( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ) batch_size = _parse_batch_size(batch_size, model, use_data_loader) @@ -859,11 +875,16 @@ def compute_embeddings( if samples.media_type == fom.VIDEO and model.media_type == "image": if batch_size is not None: return _compute_frame_embeddings_batch( - samples, model, embeddings_field, batch_size, skip_failures + samples, + model, + embeddings_field, + batch_size, + skip_failures, + progress, ) return _compute_frame_embeddings_single( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ) if use_data_loader: @@ -874,27 +895,33 @@ def compute_embeddings( batch_size, num_workers, skip_failures, + progress, ) if batch_size is not None: return _compute_image_embeddings_batch( - samples, model, embeddings_field, batch_size, skip_failures + samples, + model, + embeddings_field, + batch_size, + skip_failures, + progress, ) return _compute_image_embeddings_single( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ) def _compute_image_embeddings_single( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ): samples = samples.select_fields() embeddings = [] errors = False - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): embedding = None @@ -924,7 +951,7 @@ def _compute_image_embeddings_single( def _compute_image_embeddings_batch( - samples, model, embeddings_field, batch_size, skip_failures + samples, model, embeddings_field, batch_size, skip_failures, progress ): samples = samples.select_fields() samples_loader = fou.iter_batches(samples, batch_size) @@ -932,7 +959,7 @@ def _compute_image_embeddings_batch( embeddings = [] errors = False - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample_batch in samples_loader: embeddings_batch = [None] * len(sample_batch) @@ -970,7 +997,13 @@ def _compute_image_embeddings_batch( def _compute_image_embeddings_data_loader( - samples, model, embeddings_field, batch_size, num_workers, skip_failures + samples, + model, + embeddings_field, + batch_size, + num_workers, + skip_failures, + progress, ): samples = samples.select_fields() samples_loader = fou.iter_batches(samples, batch_size) @@ -981,7 +1014,7 @@ def _compute_image_embeddings_data_loader( embeddings = [] errors = False - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample_batch, imgs in zip(samples_loader, data_loader): embeddings_batch = [None] * len(sample_batch) @@ -1021,7 +1054,7 @@ def _compute_image_embeddings_data_loader( def _compute_frame_embeddings_single( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ): samples = samples.select_fields() frame_counts, total_frame_count = _get_frame_counts(samples) @@ -1029,7 +1062,7 @@ def _compute_frame_embeddings_single( embeddings_dict = {} - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): embeddings = [] @@ -1080,7 +1113,7 @@ def _compute_frame_embeddings_single( def _compute_frame_embeddings_batch( - samples, model, embeddings_field, batch_size, skip_failures + samples, model, embeddings_field, batch_size, skip_failures, progress ): samples = samples.select_fields() frame_counts, total_frame_count = _get_frame_counts(samples) @@ -1088,7 +1121,7 @@ def _compute_frame_embeddings_batch( embeddings_dict = {} - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): embeddings = [] @@ -1143,14 +1176,16 @@ def _compute_frame_embeddings_batch( return embeddings_dict -def _compute_video_embeddings(samples, model, embeddings_field, skip_failures): +def _compute_video_embeddings( + samples, model, embeddings_field, skip_failures, progress +): samples = samples.select_fields() is_clips = samples._dataset._is_clips embeddings = [] errors = False - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): if is_clips: frames = etaf.FrameRange(*sample.support) @@ -1196,6 +1231,7 @@ def compute_patch_embeddings( batch_size=None, num_workers=None, skip_failures=True, + progress=None, ): """Computes embeddings for the image patches defined by ``patches_field`` of the samples in the collection using the given :class:`Model`. @@ -1246,6 +1282,7 @@ def compute_patch_embeddings( Only applicable for Torch models skip_failures (True): whether to gracefully continue without raising an error if embeddings cannot be generated for a sample + progress (None): whether to render a progress bar Returns: one of the following: @@ -1359,6 +1396,7 @@ def compute_patch_embeddings( handle_missing, batch_size, skip_failures, + progress, ) if use_data_loader: @@ -1373,6 +1411,7 @@ def compute_patch_embeddings( batch_size, num_workers, skip_failures, + progress, ) return _embed_patches( @@ -1385,6 +1424,7 @@ def compute_patch_embeddings( handle_missing, batch_size, skip_failures, + progress, ) @@ -1398,6 +1438,7 @@ def _embed_patches( handle_missing, batch_size, skip_failures, + progress, ): samples = samples.select_fields(patches_field) @@ -1406,7 +1447,7 @@ def _embed_patches( else: embeddings_dict = {} - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): embeddings = None @@ -1494,6 +1535,7 @@ def _embed_patches_data_loader( batch_size, num_workers, skip_failures, + progress, ): samples = samples.select_fields(patches_field) @@ -1513,7 +1555,7 @@ def _embed_patches_data_loader( else: embeddings_dict = {} - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample, patches in pb(zip(samples, data_loader)): embeddings = None @@ -1561,6 +1603,7 @@ def _embed_frame_patches( handle_missing, batch_size, skip_failures, + progress, ): _patches_field = samples._FRAMES_PREFIX + patches_field samples = samples.select_fields(_patches_field) @@ -1572,7 +1615,7 @@ def _embed_frame_patches( else: embeddings_dict = {} - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): if is_clips: frames = etaf.FrameRange(*sample.support) diff --git a/fiftyone/core/odm/database.py b/fiftyone/core/odm/database.py index ef0da2e705..7d5459b3bf 100644 --- a/fiftyone/core/odm/database.py +++ b/fiftyone/core/odm/database.py @@ -648,9 +648,7 @@ def export_collection( to the document's ID num_docs (None): the total number of documents. If omitted, this must be computable via ``len(docs)`` - progress (None): whether to render a progress bar tracking the - progress. None uses the global setting, True or False overwrite - the value for the current method. + progress (None): whether to render a progress bar """ if num_docs is None: num_docs = len(docs) @@ -757,8 +755,7 @@ def insert_documents(docs, coll, ordered=False, progress=None, num_docs=None): docs: an iterable of BSON document dicts coll: a pymongo collection ordered (False): whether the documents must be inserted in order - progress (None): whether to render a progress bar tracking the - insertion + progress (None): whether to render a progress bar num_docs (None): the total number of documents. Only used when ``progress=True``. If omitted, this will be computed via ``len(docs)``, if possible diff --git a/fiftyone/core/utils.py b/fiftyone/core/utils.py index 4fa62bf75d..01e2602026 100644 --- a/fiftyone/core/utils.py +++ b/fiftyone/core/utils.py @@ -942,7 +942,7 @@ def __exit__(self, *args): class ProgressBar(etau.ProgressBar): """.. autoclass:: eta.core.utils.ProgressBar""" - def __init__(self, *args, progress=None, quiet=None, **kwargs): + def __init__(self, total=None, progress=None, quiet=None, **kwargs): if quiet is not None: # Allow overwrite with expected progress attribute progress = not quiet @@ -957,12 +957,31 @@ def __init__(self, *args, progress=None, quiet=None, **kwargs): if "iters_str" not in kwargs: kwargs["iters_str"] = "samples" + if progress: + kwargs["total"] = total + else: + kwargs["total"] = None + # For progress bars in notebooks, use a fixed size so that they will # read well across browsers, in HTML format, etc if foc.is_notebook_context() and "max_width" not in kwargs: kwargs["max_width"] = 90 - super().__init__(*args, **kwargs) + super().__init__(**kwargs) + + def __call__(self, iterable): + # Don't compute `len(iterable)` unless necessary + no_len = self._quiet and self._total is None + + if no_len: + self._total = -1 + + super().__call__(iterable) + + if no_len: + self._total = None + + return self class Batcher(abc.ABC): @@ -982,6 +1001,9 @@ def __init__( if not isinstance(iterable, foc.SampleCollection): return_views = False + if progress is None: + progress = fo.config.show_progress_bars + self.iterable = iterable self.return_views = return_views self.progress = progress diff --git a/fiftyone/core/view.py b/fiftyone/core/view.py index 1e1a9864d2..7be74d72e3 100644 --- a/fiftyone/core/view.py +++ b/fiftyone/core/view.py @@ -431,7 +431,7 @@ def view(self): """ return copy(self) - def iter_samples(self, progress=None, autosave=False, batch_size=None): + def iter_samples(self, progress=False, autosave=False, batch_size=None): """Returns an iterator over the samples in the view. Examples:: @@ -466,8 +466,7 @@ def make_label(): sample.ground_truth.label = make_label() Args: - progress (None): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -477,10 +476,13 @@ def make_label(): Returns: an iterator over :class:`fiftyone.core.sample.SampleView` instances """ + if progress is None: + progress = False + with contextlib.ExitStack() as exit_context: samples = self._iter_samples() - pb = fou.ProgressBar(total=len(self), progress=progress) + pb = fou.ProgressBar(total=self, progress=progress) exit_context.enter_context(pb) samples = pb(samples) @@ -539,7 +541,7 @@ def make_sample(d): def iter_groups( self, group_slices=None, - progress=None, + progress=False, autosave=False, batch_size=None, ): @@ -581,8 +583,7 @@ def make_label(): Args: group_slices (None): an optional subset of group slices to load - progress (None): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -601,10 +602,13 @@ def make_label(): "Use iter_dynamic_groups() for dynamic group views" ) + if progress is None: + progress = False + with contextlib.ExitStack() as exit_context: groups = self._iter_groups(group_slices=group_slices) - pb = fou.ProgressBar(total=len(self), progress=progress) + pb = fou.ProgressBar(total=self, progress=progress) exit_context.enter_context(pb) groups = pb(groups) @@ -677,8 +681,7 @@ def iter_dynamic_groups(self, progress=False): print("%s: %d" % (group_value, len(group))) Args: - progress (False): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar Returns: an iterator that emits :class:`DatasetView` instances, one per @@ -687,13 +690,15 @@ def iter_dynamic_groups(self, progress=False): if not self._is_dynamic_groups: raise ValueError("%s does not contain dynamic groups" % type(self)) + if progress is None: + progress = False + with contextlib.ExitStack() as context: groups = self._iter_dynamic_groups() - if progress: - pb = fou.ProgressBar(total=len(self)) - context.enter_context(pb) - groups = pb(groups) + pb = fou.ProgressBar(total=self, progress=progress) + context.enter_context(pb) + groups = pb(groups) for group in groups: yield group diff --git a/fiftyone/utils/annotations.py b/fiftyone/utils/annotations.py index f7a3bf9851..7b3a55ec8c 100644 --- a/fiftyone/utils/annotations.py +++ b/fiftyone/utils/annotations.py @@ -1020,7 +1020,7 @@ def load_annotations( or ``None`` if there aren't any cleanup (False): whether to delete any informtation regarding this run from the annotation backend after loading the annotations - progress (None): whether to render a progress bar tracking the progress + progress (None): whether to render a progress bar **kwargs: keyword arguments for the run's :meth:`fiftyone.core.annotation.AnnotationMethodConfig.load_credentials` method @@ -2343,9 +2343,7 @@ def draw_labeled_images( If omitted, all compatiable fields are rendered config (None): an optional :class:`DrawConfig` configuring how to draw the labels - progress (None): whether to render a progress bar tracking the - progress. None uses the global setting, True or False overwrite - the value for the current method. + progress (None): whether to render a progress bar **kwargs: optional keyword arguments specifying parameters of the default :class:`DrawConfig` to override diff --git a/fiftyone/utils/coco.py b/fiftyone/utils/coco.py index a89151d87c..ccaa2cf185 100644 --- a/fiftyone/utils/coco.py +++ b/fiftyone/utils/coco.py @@ -1985,29 +1985,6 @@ def _load_image_ids_json(json_path): return [_id for _id in etas.load_json(json_path)] -def _make_images_list(images_dir): - logger.info("Computing image metadata for '%s'", images_dir) - - image_paths = foud.parse_images_dir(images_dir) - - images = [] - with fou.ProgressBar() as pb: - for idx, image_path in pb(enumerate(image_paths)): - metadata = fom.ImageMetadata.build_for(image_path) - images.append( - { - "id": idx, - "file_name": os.path.basename(image_path), - "height": metadata.height, - "width": metadata.width, - "license": None, - "coco_url": None, - } - ) - - return images - - def _to_labels_map_rev(classes): return {c: i for i, c in enumerate(classes)} diff --git a/fiftyone/utils/csv.py b/fiftyone/utils/csv.py index a9f3839257..a055b1ea40 100644 --- a/fiftyone/utils/csv.py +++ b/fiftyone/utils/csv.py @@ -379,12 +379,12 @@ def setup(self): self._include_media = include_media self._needs_metadata = needs_metadata - def export_samples(self, sample_collection): + def export_samples(self, sample_collection, progress=None): if self._needs_metadata: sample_collection.compute_metadata() idx = self._media_idx - with fou.ProgressBar(total=len(sample_collection)) as pb: + with fou.ProgressBar(total=sample_collection, progress=progress) as pb: for data in pb(zip(*sample_collection.values(self._paths))): data = [_parse_value(d) for d in data] diff --git a/fiftyone/utils/cvat.py b/fiftyone/utils/cvat.py index e2ad1302bf..78fdfcf770 100644 --- a/fiftyone/utils/cvat.py +++ b/fiftyone/utils/cvat.py @@ -3995,13 +3995,14 @@ def delete_project(self, project_id): if project_name is not None: self._project_id_map.pop(project_name, None) - def delete_projects(self, project_ids): + def delete_projects(self, project_ids, progress=None): """Deletes the given projects from the CVAT server. Args: project_ids: an iterable of project IDs + progress (None): whether to render a progress bar """ - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for project_id in pb(list(project_ids)): self.delete_project(project_id) @@ -4152,13 +4153,14 @@ def delete_task(self, task_id): if self.task_exists(task_id): self.delete(self.task_url(task_id)) - def delete_tasks(self, task_ids): + def delete_tasks(self, task_ids, progress=None): """Deletes the given tasks from the CVAT server. Args: task_ids: an iterable of task IDs + progress (None): whether to render a progress bar """ - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for task_id in pb(list(task_ids)): self.delete_task(task_id) diff --git a/fiftyone/utils/data/exporters.py b/fiftyone/utils/data/exporters.py index 5fce0eb30e..cc35d2e166 100644 --- a/fiftyone/utils/data/exporters.py +++ b/fiftyone/utils/data/exporters.py @@ -57,6 +57,7 @@ def export_samples( dataset_exporter=None, label_field=None, frame_labels_field=None, + progress=None, num_samples=None, **kwargs, ): @@ -192,8 +193,10 @@ def export_samples( or a dictionary mapping field names to output keys describing the frame label fields to export. Only applicable if ``dataset_exporter`` is a :class:`LabeledVideoDatasetExporter` + progress (None): whether to render a progress bar num_samples (None): the number of samples in ``samples``. If omitted, - this is computed (if possible) via ``len(samples)`` + this is computed (if possible) via ``len(samples)`` if needed for + progress tracking **kwargs: optional keyword arguments to pass to the dataset exporter's constructor. If you are exporting image patches, this can also contain keyword arguments for @@ -234,7 +237,7 @@ def export_samples( sample_collection = samples if isinstance(dataset_exporter, BatchDatasetExporter): - _write_batch_dataset(dataset_exporter, samples) + _write_batch_dataset(dataset_exporter, samples, progress=progress) return if isinstance( @@ -252,7 +255,7 @@ def export_samples( **patches_kwargs, ) sample_parser = ImageSampleParser() - num_samples = len(samples) + num_samples = samples else: sample_parser = FiftyOneUnlabeledImageSampleParser( compute_metadata=True @@ -262,7 +265,7 @@ def export_samples( if found_clips and not samples._is_clips: # Export unlabeled video clips samples = samples.to_clips(label_field) - num_samples = len(samples) + num_samples = samples # True for copy/move/symlink, False for manifest/no export _export_media = getattr( @@ -298,7 +301,7 @@ def export_samples( **patches_kwargs, ) sample_parser = ImageClassificationSampleParser() - num_samples = len(samples) + num_samples = samples else: label_fcn = _make_label_coercion_functions( label_field, samples, dataset_exporter @@ -313,7 +316,7 @@ def export_samples( if found_clips and not samples._is_clips: # Export labeled video clips samples = samples.to_clips(label_field) - num_samples = len(samples) + num_samples = samples # True for copy/move/symlink, False for manifest/no export _export_media = getattr( @@ -358,8 +361,9 @@ def export_samples( samples, sample_parser, dataset_exporter, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) @@ -367,8 +371,9 @@ def write_dataset( samples, sample_parser, dataset_exporter, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): """Writes the samples to disk as a dataset in the specified format. @@ -378,20 +383,19 @@ def write_dataset( use to parse the samples dataset_exporter: a :class:`DatasetExporter` to use to write the dataset - num_samples (None): the number of samples in ``samples``. If omitted, - this is computed (if possible) via ``len(samples)`` sample_collection (None): the :class:`fiftyone.core.collections.SampleCollection` from which ``samples`` were extracted. If ``samples`` is itself a :class:`fiftyone.core.collections.SampleCollection`, this parameter defaults to ``samples``. This parameter is optional and is only passed to :meth:`DatasetExporter.log_collection` + progress (None): whether to render a progress bar + num_samples (None): the number of samples in ``samples``. If omitted, + this is computed (if possible) via ``len(samples)`` if needed for + progress tracking """ if num_samples is None: - try: - num_samples = len(samples) - except: - pass + num_samples = samples if sample_collection is None and isinstance(samples, foc.SampleCollection): sample_collection = samples @@ -400,15 +404,17 @@ def write_dataset( _write_generic_sample_dataset( dataset_exporter, samples, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) elif isinstance(dataset_exporter, GroupDatasetExporter): _write_group_dataset( dataset_exporter, samples, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) elif isinstance( dataset_exporter, @@ -418,8 +424,9 @@ def write_dataset( dataset_exporter, samples, sample_parser, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) elif isinstance( dataset_exporter, @@ -429,16 +436,18 @@ def write_dataset( dataset_exporter, samples, sample_parser, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) elif isinstance(dataset_exporter, UnlabeledMediaDatasetExporter): _write_unlabeled_dataset( dataset_exporter, samples, sample_parser, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) else: raise ValueError( @@ -803,7 +812,7 @@ def _classification_to_detections(label): ) -def _write_batch_dataset(dataset_exporter, samples): +def _write_batch_dataset(dataset_exporter, samples, progress=None): if not isinstance(samples, foc.SampleCollection): raise ValueError( "%s can only export %s instances" @@ -811,16 +820,17 @@ def _write_batch_dataset(dataset_exporter, samples): ) with dataset_exporter: - dataset_exporter.export_samples(samples) + dataset_exporter.export_samples(samples, progress=progress) def _write_generic_sample_dataset( dataset_exporter, samples, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -838,8 +848,9 @@ def _write_generic_sample_dataset( def _write_group_dataset( dataset_exporter, samples, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): if not isinstance(samples, foc.SampleCollection): raise ValueError( @@ -853,7 +864,7 @@ def _write_group_dataset( % (type(dataset_exporter), samples.media_type) ) - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -866,12 +877,13 @@ def _write_image_dataset( dataset_exporter, samples, sample_parser, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): labeled_images = isinstance(dataset_exporter, LabeledImageDatasetExporter) - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -919,12 +931,13 @@ def _write_video_dataset( dataset_exporter, samples, sample_parser, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): labeled_videos = isinstance(dataset_exporter, LabeledVideoDatasetExporter) - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -967,10 +980,11 @@ def _write_unlabeled_dataset( dataset_exporter, samples, sample_parser, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -1377,12 +1391,13 @@ def export_sample(self, *args, **kwargs): % type(self) ) - def export_samples(self, sample_collection): + def export_samples(self, sample_collection, progress=None): """Exports the given sample collection. Args: sample_collection: a :class:`fiftyone.core.collections.SampleCollection` + progress (None): whether to render a progress bar """ raise NotImplementedError("subclass must implement export_samples()") @@ -2052,7 +2067,7 @@ def setup(self): ) self._media_exporter.setup() - def export_samples(self, sample_collection): + def export_samples(self, sample_collection, progress=None): etau.ensure_dir(self.export_dir) if sample_collection.media_type == fomm.GROUP: @@ -2102,6 +2117,7 @@ def _prep_sample(sd): self._samples_path, key="samples", patt=patt, + progress=progress, num_docs=num_samples, ) @@ -2129,6 +2145,7 @@ def _prep_sample(sd): key="frames", patt=patt, num_docs=num_frames, + progress=progress, ) dataset = sample_collection._dataset diff --git a/fiftyone/utils/data/importers.py b/fiftyone/utils/data/importers.py index 999eeb01bf..912f7a141d 100644 --- a/fiftyone/utils/data/importers.py +++ b/fiftyone/utils/data/importers.py @@ -57,6 +57,7 @@ def import_samples( expand_schema=True, dynamic=False, add_info=True, + progress=None, ): """Adds the samples from the given :class:`DatasetImporter` to the dataset. @@ -87,6 +88,7 @@ def import_samples( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -114,7 +116,9 @@ def import_samples( ) with dataset_importer: - return dataset_importer.import_samples(dataset, tags=tags) + return dataset_importer.import_samples( + dataset, tags=tags, progress=progress + ) # # Non-batch imports @@ -130,11 +134,6 @@ def import_samples( dynamic, ) - try: - num_samples = len(dataset_importer) - except: - num_samples = None - if isinstance(dataset_importer, GroupDatasetImporter): samples = _generate_group_samples(dataset_importer, parse_sample) else: @@ -144,7 +143,8 @@ def import_samples( samples, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, + progress=progress, + num_samples=dataset_importer, ) if add_info and dataset_importer.has_dataset_info: @@ -174,6 +174,7 @@ def merge_samples( expand_schema=True, dynamic=False, add_info=True, + progress=None, ): """Merges the samples from the given :class:`DatasetImporter` into the dataset. @@ -265,6 +266,7 @@ def merge_samples( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset + progress (None): whether to render a progress bar """ if etau.is_str(tags): tags = [tags] @@ -282,7 +284,9 @@ def merge_samples( try: with dataset_importer: - dataset_importer.import_samples(tmp, tags=tags) + dataset_importer.import_samples( + tmp, tags=tags, progress=progress + ) dataset.merge_samples( tmp, @@ -317,11 +321,6 @@ def merge_samples( dynamic, ) - try: - num_samples = len(dataset_importer) - except: - num_samples = None - if isinstance(dataset_importer, GroupDatasetImporter): samples = _generate_group_samples(dataset_importer, parse_sample) else: @@ -339,7 +338,8 @@ def merge_samples( overwrite=overwrite, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, + progress=progress, + num_samples=dataset_importer, ) if add_info and dataset_importer.has_dataset_info: @@ -982,12 +982,13 @@ def __next__(self): def has_dataset_info(self): return False - def import_samples(self, dataset, tags=None): + def import_samples(self, dataset, tags=None, progress=None): """Imports the samples into the given dataset. Args: dataset: a :class:`fiftyone.core.dataset.Dataset` tags (None): an optional list of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -1764,7 +1765,7 @@ def setup(self): else: self._has_frames = False - def import_samples(self, dataset, tags=None): + def import_samples(self, dataset, tags=None, progress=None): dataset_dict = foo.import_document(self._metadata_path) if len(dataset) > 0 and fomi.needs_migration( @@ -1778,7 +1779,7 @@ def import_samples(self, dataset, tags=None): try: sample_ids = self._import_samples( - tmp_dataset, dataset_dict, tags=tags + tmp_dataset, dataset_dict, tags=tags, progress=progress ) dataset.add_collection(tmp_dataset) finally: @@ -1786,9 +1787,11 @@ def import_samples(self, dataset, tags=None): return sample_ids - return self._import_samples(dataset, dataset_dict, tags=tags) + return self._import_samples( + dataset, dataset_dict, tags=tags, progress=progress + ) - def _import_samples(self, dataset, dataset_dict, tags=None): + def _import_samples(self, dataset, dataset_dict, tags=None, progress=None): name = dataset.name empty_import = not bool(dataset) @@ -1896,7 +1899,7 @@ def _parse_sample(sd): map(_parse_sample, samples), dataset._sample_collection, ordered=self.ordered, - progress=True, + progress=progress, num_docs=num_samples, ) @@ -1924,7 +1927,7 @@ def _parse_frame(fd): map(_parse_frame, frames), dataset._frame_collection, ordered=self.ordered, - progress=True, + progress=progress, num_docs=num_frames, ) diff --git a/fiftyone/utils/data/parsers.py b/fiftyone/utils/data/parsers.py index ac98d3e47d..20bbf0a5f5 100644 --- a/fiftyone/utils/data/parsers.py +++ b/fiftyone/utils/data/parsers.py @@ -24,7 +24,7 @@ import fiftyone.utils.video as fouv -def add_images(dataset, samples, sample_parser, tags=None): +def add_images(dataset, samples, sample_parser, tags=None, progress=None): """Adds the given images to the dataset. This operation does not read the images. @@ -40,6 +40,7 @@ def add_images(dataset, samples, sample_parser, tags=None): parse the samples tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -76,14 +77,12 @@ def parse_sample(sample): return Sample(filepath=image_path, metadata=metadata, tags=tags) - try: - num_samples = len(samples) - except: - num_samples = None - _samples = map(parse_sample, samples) return dataset.add_samples( - _samples, num_samples=num_samples, expand_schema=False + _samples, + expand_schema=False, + progress=progress, + num_samples=samples, ) @@ -95,6 +94,7 @@ def add_labeled_images( tags=None, expand_schema=True, dynamic=False, + progress=None, ): """Adds the given labeled images to the dataset. @@ -127,6 +127,7 @@ def add_labeled_images( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -191,21 +192,17 @@ def parse_sample(sample): dataset._ensure_label_field(label_field, sample_parser.label_cls) expand_schema = False - try: - num_samples = len(samples) - except: - num_samples = None - _samples = map(parse_sample, samples) return dataset.add_samples( _samples, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, + progress=progress, + num_samples=samples, ) -def add_videos(dataset, samples, sample_parser, tags=None): +def add_videos(dataset, samples, sample_parser, tags=None, progress=None): """Adds the given videos to the dataset. This operation does not read the videos. @@ -221,6 +218,7 @@ def add_videos(dataset, samples, sample_parser, tags=None): parse the samples tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -251,16 +249,13 @@ def parse_sample(sample): return Sample(filepath=video_path, metadata=metadata, tags=tags) - try: - num_samples = len(samples) - except: - num_samples = None - - _samples = map(parse_sample, samples) - # @todo: skip schema expansion and set media type before adding samples + _samples = map(parse_sample, samples) return dataset.add_samples( - _samples, num_samples=num_samples, expand_schema=True + _samples, + expand_schema=True, + progress=progress, + num_samples=samples, ) @@ -272,6 +267,7 @@ def add_labeled_videos( tags=None, expand_schema=True, dynamic=False, + progress=None, ): """Adds the given labeled videos to the dataset. @@ -303,6 +299,7 @@ def add_labeled_videos( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar Returns: a list of IDs of the samples that were added to the dataset @@ -365,17 +362,13 @@ def parse_sample(sample): return sample - try: - num_samples = len(samples) - except: - num_samples = None - _samples = map(parse_sample, samples) return dataset.add_samples( _samples, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, + progress=progress, + num_samples=samples, ) diff --git a/fiftyone/utils/eval/activitynet.py b/fiftyone/utils/eval/activitynet.py index dd813b682d..d15fa07ee6 100644 --- a/fiftyone/utils/eval/activitynet.py +++ b/fiftyone/utils/eval/activitynet.py @@ -123,7 +123,13 @@ def evaluate(self, sample, eval_key=None): ) def generate_results( - self, samples, matches, eval_key=None, classes=None, missing=None + self, + samples, + matches, + eval_key=None, + classes=None, + missing=None, + progress=None, ): """Generates aggregate evaluation results for the samples. @@ -146,6 +152,7 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched segments are given this label for results purposes + progress (None): whether to render a progress bar Returns: a :class:`DetectionResults` @@ -176,7 +183,7 @@ def generate_results( # IoU sweep logger.info("Performing IoU sweep...") - for sample in _samples.iter_samples(progress=True): + for sample in _samples.iter_samples(progress=progress): # Don't edit user's data during sweep gts = _copy_labels(sample[self.gt_field]) preds = _copy_labels(sample[self.pred_field]) diff --git a/fiftyone/utils/eval/classification.py b/fiftyone/utils/eval/classification.py index c89c689141..57f73ea91f 100644 --- a/fiftyone/utils/eval/classification.py +++ b/fiftyone/utils/eval/classification.py @@ -35,6 +35,7 @@ def evaluate_classifications( classes=None, missing=None, method=None, + progress=None, **kwargs, ): """Evaluates the classification predictions in the given collection with @@ -81,6 +82,7 @@ def evaluate_classifications( supported values are ``fo.evaluation_config.classification_backends.keys()`` and the default is ``fo.evaluation_config.default_classification_backend`` + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for the constructor of the :class:`ClassificationEvaluationConfig` being used @@ -99,7 +101,11 @@ def evaluate_classifications( eval_method.register_samples(samples, eval_key) results = eval_method.evaluate_samples( - samples, eval_key=eval_key, classes=classes, missing=missing + samples, + eval_key=eval_key, + classes=classes, + missing=missing, + progress=progress, ) eval_method.save_run_results(samples, eval_key, results) @@ -148,7 +154,7 @@ def register_samples(self, samples, eval_key): raise NotImplementedError("subclass must implement register_samples()") def evaluate_samples( - self, samples, eval_key=None, classes=None, missing=None + self, samples, eval_key=None, classes=None, missing=None, progress=None ): """Evaluates the predicted classifications in the given samples with respect to the specified ground truth labels. @@ -161,6 +167,7 @@ def evaluate_samples( purposes missing (None): a missing label string. Any None-valued labels are given this label for results purposes + progress (None): whether to render a progress bar Returns: a :class:`ClassificationResults` instance @@ -256,7 +263,7 @@ def register_samples(self, samples, eval_key): dataset.add_sample_field(eval_key, fof.BooleanField) def evaluate_samples( - self, samples, eval_key=None, classes=None, missing=None + self, samples, eval_key=None, classes=None, missing=None, progress=None ): pred_field = self.config.pred_field gt_field = self.config.gt_field @@ -371,7 +378,7 @@ def register_samples(self, samples, eval_key): dataset.add_sample_field(eval_key, fof.BooleanField) def evaluate_samples( - self, samples, eval_key=None, classes=None, missing=None + self, samples, eval_key=None, classes=None, missing=None, progress=None ): if classes is None: raise ValueError( @@ -573,7 +580,7 @@ def register_samples(self, samples, eval_key): dataset.add_sample_field(eval_key, fof.StringField) def evaluate_samples( - self, samples, eval_key=None, classes=None, missing=None + self, samples, eval_key=None, classes=None, missing=None, progress=None ): if classes is None or len(classes) != 2: raise ValueError( diff --git a/fiftyone/utils/eval/coco.py b/fiftyone/utils/eval/coco.py index cde9d5cf23..cd9a2508b1 100644 --- a/fiftyone/utils/eval/coco.py +++ b/fiftyone/utils/eval/coco.py @@ -166,7 +166,13 @@ def evaluate(self, sample_or_frame, eval_key=None): return _coco_evaluation_single_iou(gts, preds, eval_key, self.config) def generate_results( - self, samples, matches, eval_key=None, classes=None, missing=None + self, + samples, + matches, + eval_key=None, + classes=None, + missing=None, + progress=None, ): """Generates aggregate evaluation results for the samples. @@ -188,6 +194,7 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched objects are given this label for results purposes + progress (None): whether to render a progress bar Returns: a :class:`DetectionResults` @@ -209,7 +216,9 @@ def generate_results( thresholds, iou_threshs, classes, - ) = _compute_pr_curves(samples, self.config, classes=classes) + ) = _compute_pr_curves( + samples, self.config, classes=classes, progress=progress + ) return COCODetectionResults( samples, @@ -633,7 +642,7 @@ def _compute_matches( return matches -def _compute_pr_curves(samples, config, classes=None): +def _compute_pr_curves(samples, config, classes=None, progress=None): gt_field = config.gt_field pred_field = config.pred_field iou_threshs = config.iou_threshs @@ -650,7 +659,7 @@ def _compute_pr_curves(samples, config, classes=None): _classes = set() logger.info("Performing IoU sweep...") - for sample in samples.iter_samples(progress=True): + for sample in samples.iter_samples(progress=progress): if processing_frames: images = sample.frames.values() else: diff --git a/fiftyone/utils/eval/detection.py b/fiftyone/utils/eval/detection.py index 3b6e0a0be0..3932e12941 100644 --- a/fiftyone/utils/eval/detection.py +++ b/fiftyone/utils/eval/detection.py @@ -40,6 +40,7 @@ def evaluate_detections( use_boxes=False, classwise=True, dynamic=True, + progress=None, **kwargs, ): """Evaluates the predicted detections in the given samples with respect to @@ -131,6 +132,7 @@ def evaluate_detections( label (True) or allow matches between classes (False) dynamic (True): whether to declare the dynamic object-level attributes that are populated on the dataset's schema + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for the constructor of the :class:`DetectionEvaluationConfig` being used @@ -182,7 +184,7 @@ def evaluate_detections( matches = [] logger.info("Evaluating detections...") - for sample in _samples.iter_samples(progress=True): + for sample in _samples.iter_samples(progress=progress): if processing_frames: docs = sample.frames.values() else: @@ -211,7 +213,12 @@ def evaluate_detections( sample.save() results = eval_method.generate_results( - samples, matches, eval_key=eval_key, classes=classes, missing=missing + samples, + matches, + eval_key=eval_key, + classes=classes, + missing=missing, + progress=progress, ) eval_method.save_run_results(samples, eval_key, results) @@ -359,7 +366,13 @@ def evaluate(self, doc, eval_key=None): raise NotImplementedError("subclass must implement evaluate()") def generate_results( - self, samples, matches, eval_key=None, classes=None, missing=None + self, + samples, + matches, + eval_key=None, + classes=None, + missing=None, + progress=None, ): """Generates aggregate evaluation results for the samples. @@ -378,6 +391,7 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched objects are given this label for results purposes + progress (None): whether to render a progress bar Returns: a :class:`DetectionResults` diff --git a/fiftyone/utils/eval/openimages.py b/fiftyone/utils/eval/openimages.py index 6eecae7697..65599ef938 100644 --- a/fiftyone/utils/eval/openimages.py +++ b/fiftyone/utils/eval/openimages.py @@ -219,7 +219,13 @@ def evaluate(self, sample_or_frame, eval_key=None): ) def generate_results( - self, samples, matches, eval_key=None, classes=None, missing=None + self, + samples, + matches, + eval_key=None, + classes=None, + missing=None, + progress=None, ): """Generates aggregate evaluation results for the samples. @@ -238,6 +244,7 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched objects are given this label for results purposes + progress (None): whether to render a progress bar Returns: a :class:`OpenImagesDetectionResults` diff --git a/fiftyone/utils/eval/regression.py b/fiftyone/utils/eval/regression.py index 52534282a6..61d6995afe 100644 --- a/fiftyone/utils/eval/regression.py +++ b/fiftyone/utils/eval/regression.py @@ -36,6 +36,7 @@ def evaluate_regressions( eval_key=None, missing=None, method=None, + progress=None, **kwargs, ): """Evaluates the regression predictions in the given collection with @@ -74,6 +75,7 @@ def evaluate_regressions( supported values are ``fo.evaluation_config.regression_backends.keys()`` and the default is ``fo.evaluation_config.default_regression_backend`` + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for the constructor of the :class:`RegressionEvaluationConfig` being used @@ -92,7 +94,7 @@ def evaluate_regressions( eval_method.register_samples(samples, eval_key) results = eval_method.evaluate_samples( - samples, eval_key=eval_key, missing=missing + samples, eval_key=eval_key, missing=missing, progress=progress ) eval_method.save_run_results(samples, eval_key, results) @@ -149,7 +151,9 @@ def register_samples(self, samples, eval_key): else: dataset.add_sample_field(eval_key, fof.FloatField) - def evaluate_samples(self, samples, eval_key=None, missing=None): + def evaluate_samples( + self, samples, eval_key=None, missing=None, progress=None + ): """Evaluates the regression predictions in the given samples with respect to the specified ground truth values. @@ -158,6 +162,7 @@ def evaluate_samples(self, samples, eval_key=None, missing=None): eval_key (None): an evaluation key for this evaluation missing (None): a missing value. Any None-valued regressions are given this value for results purposes + progress (None): whether to render a progress bar Returns: a :class:`RegressionResults` instance @@ -243,7 +248,9 @@ class SimpleEvaluation(RegressionEvaluation): config: a :class:`SimpleEvaluationConfig` """ - def evaluate_samples(self, samples, eval_key=None, missing=None): + def evaluate_samples( + self, samples, eval_key=None, missing=None, progress=None + ): metric = self.config._metric if metric == "squared_error": diff --git a/fiftyone/utils/eval/segmentation.py b/fiftyone/utils/eval/segmentation.py index 2f2c5b8e35..f441a39c36 100644 --- a/fiftyone/utils/eval/segmentation.py +++ b/fiftyone/utils/eval/segmentation.py @@ -36,6 +36,7 @@ def evaluate_segmentations( eval_key=None, mask_targets=None, method=None, + progress=None, **kwargs, ): """Evaluates the specified semantic segmentation masks in the given @@ -86,6 +87,7 @@ def evaluate_segmentations( supported values are ``fo.evaluation_config.segmentation_backends.keys()`` and the default is ``fo.evaluation_config.default_segmentation_backend`` + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for the constructor of the :class:`SegmentationEvaluationConfig` being used @@ -104,7 +106,10 @@ def evaluate_segmentations( eval_method.register_samples(samples, eval_key) results = eval_method.evaluate_samples( - samples, eval_key=eval_key, mask_targets=mask_targets + samples, + eval_key=eval_key, + mask_targets=mask_targets, + progress=progress, ) eval_method.save_run_results(samples, eval_key, results) @@ -177,7 +182,9 @@ def register_samples(self, samples, eval_key): if processing_frames: dataset.add_frame_field(dice_field, fof.FloatField) - def evaluate_samples(self, samples, eval_key=None, mask_targets=None): + def evaluate_samples( + self, samples, eval_key=None, mask_targets=None, progress=None + ): """Evaluates the predicted segmentation masks in the given samples with respect to the specified ground truth masks. @@ -188,6 +195,7 @@ def evaluate_samples(self, samples, eval_key=None, mask_targets=None): contain a subset of the possible classes if you wish to evaluate a subset of the semantic classes. By default, the observed pixel values are used as labels + progress (None): whether to render a progress bar Returns: a :class:`SegmentationResults` instance @@ -311,7 +319,9 @@ class SimpleEvaluation(SegmentationEvaluation): config: a :class:`SimpleEvaluationConfig` """ - def evaluate_samples(self, samples, eval_key=None, mask_targets=None): + def evaluate_samples( + self, samples, eval_key=None, mask_targets=None, progress=None + ): pred_field = self.config.pred_field gt_field = self.config.gt_field @@ -324,7 +334,9 @@ def evaluate_samples(self, samples, eval_key=None, mask_targets=None): values, classes = zip(*sorted(mask_targets.items())) else: logger.info("Computing possible mask values...") - values, classes = _get_mask_values(samples, pred_field, gt_field) + values, classes = _get_mask_values( + samples, pred_field, gt_field, progress=progress + ) _samples = samples.select_fields([gt_field, pred_field]) pred_field, processing_frames = samples._handle_frame_field(pred_field) @@ -345,7 +357,7 @@ def evaluate_samples(self, samples, eval_key=None, mask_targets=None): dice_field = "%s_dice" % eval_key logger.info("Evaluating segmentations...") - for sample in _samples.iter_samples(progress=True): + for sample in _samples.iter_samples(progress=progress): if processing_frames: images = sample.frames.values() else: @@ -590,7 +602,7 @@ def _compute_accuracy_precision_recall(confusion_matrix, values, average): return metrics["accuracy"], metrics["precision"], metrics["recall"] -def _get_mask_values(samples, pred_field, gt_field): +def _get_mask_values(samples, pred_field, gt_field, progress=None): _samples = samples.select_fields([gt_field, pred_field]) pred_field, processing_frames = samples._handle_frame_field(pred_field) gt_field, _ = samples._handle_frame_field(gt_field) @@ -598,7 +610,7 @@ def _get_mask_values(samples, pred_field, gt_field): values = set() is_rgb = False - for sample in _samples.iter_samples(progress=True): + for sample in _samples.iter_samples(progress=progress): if processing_frames: images = sample.frames.values() else: diff --git a/fiftyone/utils/geojson.py b/fiftyone/utils/geojson.py index 6cf44b31f6..6391c1968b 100644 --- a/fiftyone/utils/geojson.py +++ b/fiftyone/utils/geojson.py @@ -24,7 +24,11 @@ def load_location_data( - samples, geojson_or_path, location_field=None, skip_missing=True + samples, + geojson_or_path, + location_field=None, + skip_missing=True, + progress=None, ): """Loads geolocation data for the given samples from the given GeoJSON data. @@ -86,6 +90,7 @@ def load_location_data( used, else a new "location" field is created skip_missing (True): whether to skip GeoJSON features with no ``filename`` properties (True) or raise an error (False) + progress (None): whether to render a progress bar """ if location_field is None: try: @@ -137,7 +142,7 @@ def load_location_data( logger.info("Loading location data for %d samples...", len(found_keys)) _samples = samples.select_fields(location_field) - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for key in pb(found_keys): sample_id = lookup[key] geometry = geometries[key] diff --git a/fiftyone/utils/image.py b/fiftyone/utils/image.py index 44c02ce820..aad5a947e6 100644 --- a/fiftyone/utils/image.py +++ b/fiftyone/utils/image.py @@ -59,6 +59,7 @@ def reencode_images( delete_originals=False, num_workers=None, skip_failures=False, + progress=None, ): """Re-encodes the images in the sample collection to the given format. @@ -100,6 +101,7 @@ def reencode_images( num_workers (None): a suggested number of worker processes to use skip_failures (False): whether to gracefully continue without raising an error if an image cannot be re-encoded + progress (None): whether to render a progress bar """ fov.validate_image_collection(sample_collection) @@ -115,6 +117,7 @@ def reencode_images( delete_originals=delete_originals, num_workers=num_workers, skip_failures=skip_failures, + progress=progress, ) @@ -134,6 +137,7 @@ def transform_images( delete_originals=False, num_workers=None, skip_failures=False, + progress=None, ): """Transforms the images in the sample collection according to the provided parameters. @@ -189,6 +193,7 @@ def transform_images( num_workers (None): a suggested number of worker processes to use skip_failures (False): whether to gracefully continue without raising an error if an image cannot be transformed + progress (None): whether to render a progress bar """ fov.validate_image_collection(sample_collection) @@ -208,6 +213,7 @@ def transform_images( delete_originals=delete_originals, num_workers=num_workers, skip_failures=skip_failures, + progress=progress, ) @@ -271,6 +277,7 @@ def _transform_images( delete_originals=False, num_workers=None, skip_failures=False, + progress=None, ): ext = _parse_ext(ext) @@ -292,6 +299,7 @@ def _transform_images( update_filepaths=update_filepaths, delete_originals=delete_originals, skip_failures=skip_failures, + progress=progress, ) else: _transform_images_multi( @@ -310,6 +318,7 @@ def _transform_images( update_filepaths=update_filepaths, delete_originals=delete_originals, skip_failures=skip_failures, + progress=progress, ) @@ -328,6 +337,7 @@ def _transform_images_single( update_filepaths=True, delete_originals=False, skip_failures=False, + progress=None, ): if output_field is None: output_field = media_field @@ -336,7 +346,7 @@ def _transform_images_single( view = sample_collection.select_fields(media_field) - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(view): inpath = sample[media_field] @@ -380,6 +390,7 @@ def _transform_images_multi( update_filepaths=True, delete_originals=False, skip_failures=False, + progress=None, ): if output_field is None: output_field = media_field @@ -413,7 +424,7 @@ def _transform_images_multi( outpaths = {} try: - with fou.ProgressBar(inputs) as pb: + with fou.ProgressBar(inputs, progress=progress) as pb: with fou.get_multiprocessing_context().Pool( processes=num_workers ) as pool: diff --git a/fiftyone/utils/iou.py b/fiftyone/utils/iou.py index d75b0e8314..fff7c1a584 100644 --- a/fiftyone/utils/iou.py +++ b/fiftyone/utils/iou.py @@ -149,6 +149,7 @@ def compute_max_ious( other_field=None, iou_attr="max_iou", id_attr=None, + progress=None, **kwargs, ): """Populates an attribute on each label in the given spatial field(s) that @@ -175,6 +176,7 @@ def compute_max_ious( iou_attr ("max_iou"): the label attribute in which to store the max IoU id_attr (None): an optional attribute in which to store the label ID of the maximum overlapping label + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for :func:`compute_ious` """ if other_field is None: @@ -202,7 +204,7 @@ def compute_max_ious( label_ids1 = [] label_ids2 = [] - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): if is_frame_field: _max_ious1 = [] _max_ious2 = [] @@ -259,7 +261,12 @@ def compute_max_ious( def find_duplicates( - sample_collection, label_field, iou_thresh=0.999, method="simple", **kwargs + sample_collection, + label_field, + iou_thresh=0.999, + method="simple", + progress=None, + **kwargs, ): """Returns IDs of duplicate labels in the given field of the collection, as defined as labels with an IoU greater than a chosen threshold with another @@ -287,6 +294,7 @@ def find_duplicates( labels are duplicates method ("simple"): the duplicate removal method to use. The supported values are ``("simple", "greedy")`` + progress (None): whether to render a progress bar **kwargs: optional keyword arguments for :func:`compute_ious` Returns: @@ -306,7 +314,7 @@ def find_duplicates( dup_ids = [] - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): if is_frame_field: for frame in sample.frames.values(): _dup_ids = _find_duplicates( diff --git a/fiftyone/utils/labelbox.py b/fiftyone/utils/labelbox.py index e130d44751..0661be48d6 100644 --- a/fiftyone/utils/labelbox.py +++ b/fiftyone/utils/labelbox.py @@ -397,14 +397,15 @@ def list_datasets(self): datasets = self._client.get_datasets() return [d.uid for d in datasets] - def delete_datasets(self, dataset_ids): + def delete_datasets(self, dataset_ids, progress=None): """Deletes the given datasets from the Labelbox server. Args: dataset_ids: an iterable of dataset IDs + progress (None): whether to render a progress bar """ logger.info("Deleting datasets...") - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for dataset_id in pb(list(dataset_ids)): dataset = self._client.get_dataset(dataset_id) dataset.delete() @@ -1490,6 +1491,7 @@ def import_from_labelbox( label_prefix=None, download_dir=None, labelbox_id_field="labelbox_id", + progress=None, ): """Imports the labels from the Labelbox project into the FiftyOne dataset. @@ -1546,6 +1548,7 @@ def import_from_labelbox( samples labelbox_id_field ("labelbox_id"): the sample field to lookup/store the IDs of the Labelbox DataRows + progress (None): whether to render a progress bar """ fov.validate_collection(dataset, media_type=(fomm.IMAGE, fomm.VIDEO)) is_video = dataset.media_type == fomm.VIDEO @@ -1567,7 +1570,7 @@ def import_from_labelbox( d_list = etas.read_json(json_path) # ref: https://github.com/Labelbox/labelbox/blob/7c79b76310fa867dd38077e83a0852a259564da1/exporters/coco-exporter/coco_exporter.py#L33 - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for d in pb(d_list): labelbox_id = d["DataRow ID"] @@ -1634,6 +1637,7 @@ def export_to_labelbox( labelbox_id_field="labelbox_id", label_field=None, frame_labels_field=None, + progress=None, ): """Exports labels from the FiftyOne samples to Labelbox format. @@ -1690,6 +1694,7 @@ def export_to_labelbox( when constructing the exported frame labels By default, no frame labels are exported + progress (None): whether to render a progress bar """ fov.validate_collection( sample_collection, media_type=(fomm.IMAGE, fomm.VIDEO) @@ -1724,7 +1729,7 @@ def export_to_labelbox( etau.ensure_empty_file(ndjson_path) # Export the labels - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(sample_collection): labelbox_id = sample[labelbox_id_field] if labelbox_id is None: @@ -1792,7 +1797,10 @@ def download_labels_from_labelbox(labelbox_project, outpath=None): def upload_media_to_labelbox( - labelbox_dataset, sample_collection, labelbox_id_field="labelbox_id" + labelbox_dataset, + sample_collection, + labelbox_id_field="labelbox_id", + progress=None, ): """Uploads the raw media for the FiftyOne samples to Labelbox. @@ -1806,11 +1814,12 @@ def upload_media_to_labelbox( :class:`fiftyone.core.collections.SampleCollection` labelbox_id_field ("labelbox_id"): the sample field in which to store the IDs of the Labelbox DataRows + progress (None): whether to render a progress bar """ # @todo use `create_data_rows()` to optimize performance # @todo handle API rate limits # Reference: https://labelbox.com/docs/python-api/data-rows - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(sample_collection): try: has_id = sample[labelbox_id_field] is not None @@ -1859,15 +1868,11 @@ def upload_labels_to_labelbox( else: annos = annos_or_ndjson_path - requests = [] count = 0 for anno_batch in fou.iter_batches(annos, batch_size): count += 1 name = "%s-upload-request-%d" % (labelbox_project.name, count) - request = labelbox_project.upload_annotations(name, anno_batch) - requests.append(request) - - return requests + labelbox_project.upload_annotations(name, anno_batch) def convert_labelbox_export_to_import(inpath, outpath=None, video_outdir=None): diff --git a/fiftyone/utils/labels.py b/fiftyone/utils/labels.py index 1e986c223d..ab711c380c 100644 --- a/fiftyone/utils/labels.py +++ b/fiftyone/utils/labels.py @@ -23,6 +23,7 @@ def objects_to_segmentations( rel_dir=None, overwrite=False, save_mask_targets=False, + progress=None, ): """Converts the instance segmentations or polylines in the specified field of the collection into semantic segmentation masks. @@ -61,6 +62,7 @@ def objects_to_segmentations( if it exists save_mask_targets (False): whether to store the ``mask_targets`` on the dataset + progress (None): whether to render a progress bar """ fov.validate_collection_label_fields( sample_collection, @@ -83,7 +85,7 @@ def objects_to_segmentations( output_dir=output_dir, rel_dir=rel_dir, idempotent=False ) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -147,6 +149,7 @@ def export_segmentations( rel_dir=None, update=True, overwrite=False, + progress=None, ): """Exports the segmentations (or heatmaps) stored as in-database arrays in the specified field to images on disk. @@ -170,6 +173,7 @@ def export_segmentations( update (True): whether to delete the arrays from the database overwrite (False): whether to delete ``output_dir`` prior to exporting if it exists + progress (None): whether to render a progress bar """ fov.validate_collection_label_fields( sample_collection, in_field, (fol.Segmentation, fol.Heatmap) @@ -185,7 +189,7 @@ def export_segmentations( output_dir=output_dir, rel_dir=rel_dir, idempotent=False ) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -209,7 +213,11 @@ def export_segmentations( def import_segmentations( - sample_collection, in_field, update=True, delete_images=False + sample_collection, + in_field, + update=True, + delete_images=False, + progress=None, ): """Imports the segmentations (or heatmaps) stored on disk in the specified field to in-database arrays. @@ -224,6 +232,7 @@ def import_segmentations( :class:`fiftyone.core.labels.Heatmap` field update (True): whether to delete the image paths from the labels delete_images (False): whether to delete any imported images from disk + progress (None): whether to render a progress bar """ fov.validate_collection_label_fields( sample_collection, in_field, (fol.Segmentation, fol.Heatmap) @@ -232,7 +241,7 @@ def import_segmentations( samples = sample_collection.select_fields(in_field) in_field, processing_frames = samples._handle_frame_field(in_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -266,6 +275,7 @@ def transform_segmentations( update=True, update_mask_targets=False, overwrite=False, + progress=None, ): """Transforms the segmentations in the given field according to the provided targets map. @@ -299,6 +309,7 @@ def transform_segmentations( dataset to reflect the transformed targets overwrite (False): whether to delete ``output_dir`` prior to exporting if it exists + progress (None): whether to render a progress bar """ fov.validate_collection_label_fields( sample_collection, in_field, fol.Segmentation @@ -315,7 +326,7 @@ def transform_segmentations( output_dir=output_dir, rel_dir=rel_dir, idempotent=False ) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -365,6 +376,7 @@ def segmentations_to_detections( out_field, mask_targets=None, mask_types="stuff", + progress=None, ): """Converts the semantic segmentations masks in the specified field of the collection into :class:`fiftyone.core.labels.Detections` with instance @@ -398,6 +410,7 @@ def segmentations_to_detections( - ``"thing"`` if all classes are thing classes - a dict mapping pixel values (2D masks) or RGB hex strings (3D masks) to ``"stuff"`` or ``"thing"`` for each class + progress (None): whether to render a progress bar """ fov.validate_collection_label_fields( sample_collection, @@ -409,7 +422,7 @@ def segmentations_to_detections( in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -426,7 +439,12 @@ def segmentations_to_detections( def instances_to_polylines( - sample_collection, in_field, out_field, tolerance=2, filled=True + sample_collection, + in_field, + out_field, + tolerance=2, + filled=True, + progress=None, ): """Converts the instance segmentations in the specified field of the collection into :class:`fiftyone.core.labels.Polylines` instances. @@ -445,6 +463,7 @@ def instances_to_polylines( tolerance (2): a tolerance, in pixels, when generating approximate polylines for each region. Typical values are 1-3 pixels filled (True): whether the polylines should be filled + progress (None): whether to render a progress bar """ fov.validate_collection_label_fields( sample_collection, @@ -456,7 +475,7 @@ def instances_to_polylines( in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -479,6 +498,7 @@ def segmentations_to_polylines( mask_targets=None, mask_types="stuff", tolerance=2, + progress=None, ): """Converts the semantic segmentations masks in the specified field of the collection into :class:`fiftyone.core.labels.Polylines` instances. @@ -513,6 +533,7 @@ def segmentations_to_polylines( masks) to ``"stuff"`` or ``"thing"`` for each class tolerance (2): a tolerance, in pixels, when generating approximate polylines for each region. Typical values are 1-3 pixels + progress (None): whether to render a progress bar """ fov.validate_collection_label_fields( sample_collection, @@ -524,7 +545,7 @@ def segmentations_to_polylines( in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -542,7 +563,9 @@ def segmentations_to_polylines( ) -def classification_to_detections(sample_collection, in_field, out_field): +def classification_to_detections( + sample_collection, in_field, out_field, progress=None +): """Converts the :class:`fiftyone.core.labels.Classification` field of the collection into a :class:`fiftyone.core.labels.Detections` field containing a single detection whose bounding box spans the entire image. @@ -554,6 +577,7 @@ def classification_to_detections(sample_collection, in_field, out_field): field out_field: the name of the :class:`fiftyone.core.labels.Detections` field to populate + progress (None): whether to render a progress bar """ fov.validate_collection_label_fields( sample_collection, in_field, fol.Classification @@ -563,7 +587,7 @@ def classification_to_detections(sample_collection, in_field, out_field): in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -582,7 +606,9 @@ def classification_to_detections(sample_collection, in_field, out_field): image[out_field] = fol.Detections(detections=[detection]) -def classifications_to_detections(sample_collection, in_field, out_field): +def classifications_to_detections( + sample_collection, in_field, out_field, progress=None +): """Converts the :class:`fiftyone.core.labels.Classifications` field of the collection into a :class:`fiftyone.core.labels.Detections` field containing detections whose bounding boxes span the entire image with one detection @@ -604,7 +630,7 @@ def classifications_to_detections(sample_collection, in_field, out_field): in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: diff --git a/fiftyone/utils/scale.py b/fiftyone/utils/scale.py index 16cd8c90a6..32f76bc0d2 100644 --- a/fiftyone/utils/scale.py +++ b/fiftyone/utils/scale.py @@ -36,6 +36,7 @@ def import_from_scale( labels_dir_or_json, label_prefix=None, scale_id_field="scale_id", + progress=None, ): """Imports the Scale AI labels into the FiftyOne dataset. @@ -186,6 +187,7 @@ def import_from_scale( that are created, separated by an underscore scale_id_field ("scale_id"): the sample field to use to associate Scale task IDs with FiftyOne samples + progress (None): whether to render a progress bar """ fov.validate_collection(dataset, media_type=(fomm.IMAGE, fomm.VIDEO)) is_video = dataset.media_type == fomm.VIDEO @@ -203,7 +205,7 @@ def import_from_scale( else: label_key = lambda k: k - with fou.ProgressBar(total=len(labels)) as pb: + with fou.ProgressBar(total=len(labels), progress=progress) as pb: for task_id, task_labels in pb(labels.items()): if task_id not in id_map: logger.info( @@ -255,6 +257,7 @@ def export_to_scale( video_playback=False, label_field=None, frame_labels_field=None, + progress=None, ): """Exports labels from the FiftyOne samples to Scale AI format. @@ -393,6 +396,7 @@ def export_to_scale( when constructing the exported frame labels By default, no frame labels are exported + progress (None): whether to render a progress bar """ fov.validate_collection( sample_collection, media_type=(fomm.IMAGE, fomm.VIDEO) @@ -430,7 +434,7 @@ def export_to_scale( # Export the labels labels = {} anno_dict = {} - for sample in sample_collection.iter_samples(progress=True): + for sample in sample_collection.iter_samples(progress=progress): metadata = sample.metadata # Get frame size diff --git a/fiftyone/utils/utils3d.py b/fiftyone/utils/utils3d.py index 17d297d214..00339acf92 100644 --- a/fiftyone/utils/utils3d.py +++ b/fiftyone/utils/utils3d.py @@ -446,6 +446,7 @@ def compute_orthographic_projection_images( subsampling_rate=None, projection_normal=None, bounds=None, + progress=None, ): """Computes orthographic projection images for the point clouds in the given collection. @@ -509,6 +510,7 @@ def compute_orthographic_projection_images( to generate each map. Either element of the tuple or any/all of its values can be None, in which case a tight crop of the point cloud along the missing dimension(s) are used + progress (None): whether to render a progress bar """ if in_group_slice is None and samples.media_type == fom.GROUP: in_group_slice = _get_point_cloud_slice(samples) @@ -537,7 +539,7 @@ def compute_orthographic_projection_images( all_metadata = [] - with fou.ProgressBar(total=len(filepaths)) as pb: + with fou.ProgressBar(total=len(filepaths), progress=progress) as pb: for filepath, group in pb(zip(filepaths, groups)): image_path = filename_maker.get_output_path( filepath, output_ext=".png" diff --git a/fiftyone/utils/video.py b/fiftyone/utils/video.py index e3b36e2e49..38c1700b66 100644 --- a/fiftyone/utils/video.py +++ b/fiftyone/utils/video.py @@ -97,6 +97,7 @@ def reencode_videos( delete_originals=False, skip_failures=False, verbose=False, + progress=None, **kwargs, ): """Re-encodes the videos in the sample collection as H.264 MP4s that can be @@ -152,6 +153,7 @@ def reencode_videos( an error if a video cannot be re-encoded verbose (False): whether to log the ``ffmpeg`` commands that are executed + progress (None): whether to render a progress bar **kwargs: keyword arguments for ``eta.core.video.FFmpeg(**kwargs)`` """ fov.validate_video_collection(sample_collection) @@ -168,6 +170,7 @@ def reencode_videos( delete_originals=delete_originals, skip_failures=skip_failures, verbose=verbose, + progress=progress, **kwargs, ) @@ -190,6 +193,7 @@ def transform_videos( delete_originals=False, skip_failures=False, verbose=False, + progress=None, **kwargs, ): """Transforms the videos in the sample collection according to the provided @@ -258,6 +262,7 @@ def transform_videos( an error if a video cannot be transformed verbose (False): whether to log the ``ffmpeg`` commands that are executed + progress (None): whether to render a progress bar **kwargs: keyword arguments for ``eta.core.video.FFmpeg(**kwargs)`` """ fov.validate_video_collection(sample_collection) @@ -280,6 +285,7 @@ def transform_videos( delete_originals=delete_originals, skip_failures=skip_failures, verbose=verbose, + progress=progress, **kwargs, ) @@ -303,6 +309,7 @@ def sample_videos( delete_originals=False, skip_failures=False, verbose=False, + progress=None, **kwargs, ): """Samples the videos in the sample collection into directories of @@ -404,6 +411,7 @@ def sample_videos( an error if a video cannot be sampled verbose (False): whether to log the ``ffmpeg`` commands that are executed + progress (None): whether to render a progress bar **kwargs: keyword arguments for ``eta.core.video.FFmpeg(**kwargs)`` """ fov.validate_video_collection(sample_collection) @@ -428,6 +436,7 @@ def sample_videos( delete_originals=delete_originals, skip_failures=skip_failures, verbose=verbose, + progress=progress, **kwargs, ) @@ -725,6 +734,7 @@ def _transform_videos( delete_originals=False, skip_failures=False, verbose=False, + progress=None, **kwargs, ): if output_field is None: @@ -744,7 +754,7 @@ def _transform_videos( if frames is None: frames = itertools.repeat(None) - with fou.ProgressBar(total=len(view)) as pb: + with fou.ProgressBar(total=view, progress=progress) as pb: for sample, _frames in pb(zip(view, frames)): inpath = sample[media_field] diff --git a/tests/unittests/utils_tests.py b/tests/unittests/utils_tests.py index 3e91604ec5..91cbb6cdc7 100644 --- a/tests/unittests/utils_tests.py +++ b/tests/unittests/utils_tests.py @@ -458,7 +458,7 @@ def test_load_dataset_nonexistent(self, mock_get_db_conn): class ProgressBarTests(unittest.TestCase): def _test_correct_value(self, progress, global_progress, quiet, expected): fo.config.show_progress_bars = global_progress - with fou.ProgressBar(list(), progress=progress, quiet=quiet) as pb: + with fou.ProgressBar([], progress=progress, quiet=quiet) as pb: assert pb._progress == expected def test_progress_None_uses_global(self): @@ -484,7 +484,7 @@ def test_progress_overwrites_global(self): ) def test_quiet_overwrites_all(self): - # Careful, we expect here to have progress the opposite value of quiet, as they are opposites + # Careful, we expect here to have progress the opposite value of quiet self._test_correct_value( progress=True, global_progress=True, quiet=True, expected=False ) From 4e3ae2a2afab5c9c3c508e39ae8728feef760711 Mon Sep 17 00:00:00 2001 From: brimoor Date: Mon, 1 Jan 2024 20:31:18 -0500 Subject: [PATCH 05/12] add support for progress callbacks --- fiftyone/__public__.py | 1 + fiftyone/core/utils.py | 121 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/fiftyone/__public__.py b/fiftyone/__public__.py index 74020771ec..d40706b6ce 100644 --- a/fiftyone/__public__.py +++ b/fiftyone/__public__.py @@ -213,6 +213,7 @@ disable_progress_bars, pprint, pformat, + report_progress, ProgressBar, ) from .core.view import DatasetView diff --git a/fiftyone/core/utils.py b/fiftyone/core/utils.py index 01e2602026..58451c6be1 100644 --- a/fiftyone/core/utils.py +++ b/fiftyone/core/utils.py @@ -943,25 +943,25 @@ class ProgressBar(etau.ProgressBar): """.. autoclass:: eta.core.utils.ProgressBar""" def __init__(self, total=None, progress=None, quiet=None, **kwargs): + if progress is None: + progress = fo.config.show_progress_bars + if quiet is not None: - # Allow overwrite with expected progress attribute progress = not quiet - if progress is None: - # Use global config value - progress = fo.config.show_progress_bars + if callable(progress): + callback = progress + progress = False + else: + callback = None - self._progress = progress - kwargs["quiet"] = not self._progress + kwargs["total"] = total + if isinstance(progress, bool): + kwargs["quiet"] = not progress if "iters_str" not in kwargs: kwargs["iters_str"] = "samples" - if progress: - kwargs["total"] = total - else: - kwargs["total"] = None - # For progress bars in notebooks, use a fixed size so that they will # read well across browsers, in HTML format, etc if foc.is_notebook_context() and "max_width" not in kwargs: @@ -969,10 +969,11 @@ def __init__(self, total=None, progress=None, quiet=None, **kwargs): super().__init__(**kwargs) + self._callback = callback + def __call__(self, iterable): - # Don't compute `len(iterable)` unless necessary + # Ensure that `len(iterable)` is not computed unnecessarily no_len = self._quiet and self._total is None - if no_len: self._total = -1 @@ -983,6 +984,100 @@ def __call__(self, iterable): return self + def set_iteration(self, *args, **kwargs): + super().set_iteration(*args, **kwargs) + + if self._callback is not None: + self._callback(self) + + +def report_progress(progress, n=None, dt=None): + """Wraps the provided progress function such that it will only be called + at the specified increments or time intervals. + + Example usage:: + + import fiftyone as fo + import fiftyone.zoo as foz + + def print_progress(pb): + if pb.complete: + print("COMPLETE") + else: + print("PROGRESS: %0.3f" % pb.progress) + + dataset = foz.load_zoo_dataset("cifar10", split="test") + + # Print progress at 10 equally-spaced increments + progress = fo.report_progress(print_progress, n=10) + dataset.compute_metadata(progress=progress) + + # Print progress every 0.5 seconds + progress = fo.report_progress(print_progress, dt=0.5) + dataset.compute_metadata(progress=progress, overwrite=True) + + Args: + progress: a function that accepts a :class:`ProgressBar` as input + n (None): a number of equally-spaced increments to invoke ``progress`` + dt (None): a number of seconds between ``progress`` calls + + Returns: + a function that accepts a :class:`ProgressBar` as input + """ + if n is not None: + return _report_progress_n(progress, n) + + if dt is not None: + return _report_progress_dt(progress, dt) + + return progress + + +def _report_progress_n(progress, n): + def progress_n(pb): + if not hasattr(pb, "_next_idx"): + if pb.has_total and n > 0: + next_iters = [ + int(np.round(i)) + for i in np.linspace(0, pb.total, min(n, pb.total) + 1) + ][1:] + + pb._next_idx = 0 + pb._next_iters = next_iters + else: + pb._next_idx = None + pb._next_iters = None + + if ( + pb._next_idx is not None + and pb.iteration >= pb._next_iters[pb._next_idx] + ): + progress(pb) + + pb._next_idx += 1 + if pb._next_idx >= len(pb._next_iters): + pb._next_idx = None + + return progress_n + + +def _report_progress_dt(progress, dt): + def progress_dt(pb): + if not hasattr(pb, "_next_dt"): + pb._next_dt = dt + + if pb._next_dt is not None and ( + pb.elapsed_time >= pb._next_dt or pb.complete + ): + progress(pb) + + if not pb.complete: + pb._next_dt += dt + else: + pb._next_dt = None + + return progress_dt + class Batcher(abc.ABC): """Base class for iterating over the elements of an iterable in batches.""" From e87a72539c79062f67453e43c0b42a82bb861ad1 Mon Sep 17 00:00:00 2001 From: brimoor Date: Tue, 2 Jan 2024 00:21:07 -0500 Subject: [PATCH 06/12] progress upgrades --- fiftyone/core/storage.py | 36 +++++++++++---------------- fiftyone/core/utils.py | 18 +++++++------- fiftyone/utils/data/importers.py | 2 -- fiftyone/utils/eval/classification.py | 3 +++ fiftyone/utils/eval/detection.py | 3 +++ fiftyone/utils/eval/regression.py | 3 +++ fiftyone/utils/eval/segmentation.py | 3 +++ fiftyone/utils/labels.py | 1 + fiftyone/zoo/datasets/__init__.py | 17 +++++++++---- tests/unittests/utils_tests.py | 8 +++--- 10 files changed, 53 insertions(+), 41 deletions(-) diff --git a/fiftyone/core/storage.py b/fiftyone/core/storage.py index 68ab5322ee..7d7978b23d 100644 --- a/fiftyone/core/storage.py +++ b/fiftyone/core/storage.py @@ -689,7 +689,7 @@ def copy_file(inpath, outpath): _copy_file(inpath, outpath, cleanup=False) -def copy_files(inpaths, outpaths, skip_failures=False, progress=False): +def copy_files(inpaths, outpaths, skip_failures=False, progress=None): """Copies the files to the given locations. Args: @@ -697,14 +697,13 @@ def copy_files(inpaths, outpaths, skip_failures=False, progress=False): outpaths: a list of output paths skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar """ _copy_files(inpaths, outpaths, skip_failures, progress) def copy_dir( - indir, outdir, overwrite=True, skip_failures=False, progress=False + indir, outdir, overwrite=True, skip_failures=False, progress=None ): """Copies the input directory to the output directory. @@ -715,8 +714,7 @@ def copy_dir( or merge its contents (False) skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar """ if overwrite and os.path.isdir(outdir): delete_dir(outdir) @@ -741,7 +739,7 @@ def move_file(inpath, outpath): _copy_file(inpath, outpath, cleanup=True) -def move_files(inpaths, outpaths, skip_failures=False, progress=False): +def move_files(inpaths, outpaths, skip_failures=False, progress=None): """Moves the files to the given locations. Args: @@ -749,8 +747,7 @@ def move_files(inpaths, outpaths, skip_failures=False, progress=False): outpaths: a list of output paths skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar """ tasks = [(i, o, skip_failures) for i, o in zip(inpaths, outpaths)] if tasks: @@ -758,7 +755,7 @@ def move_files(inpaths, outpaths, skip_failures=False, progress=False): def move_dir( - indir, outdir, overwrite=True, skip_failures=False, progress=False + indir, outdir, overwrite=True, skip_failures=False, progress=None ): """Moves the contents of the given directory into the given output directory. @@ -770,8 +767,7 @@ def move_dir( or merge its contents (False) skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar """ if overwrite and os.path.isdir(outdir): delete_dir(outdir) @@ -793,7 +789,7 @@ def delete_file(path): _delete_file(path) -def delete_files(paths, skip_failures=False, progress=False): +def delete_files(paths, skip_failures=False, progress=None): """Deletes the files from the given locations. Any empty directories are also recursively deleted from the resulting @@ -803,8 +799,7 @@ def delete_files(paths, skip_failures=False, progress=False): paths: a list of paths skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar """ tasks = [(p, skip_failures) for p in paths] if tasks: @@ -821,15 +816,14 @@ def delete_dir(dirpath): etau.delete_dir(dirpath) -def run(fcn, tasks, num_workers=None, progress=False): +def run(fcn, tasks, num_workers=None, progress=None): """Applies the given function to each element of the given tasks. Args: fcn: a function that accepts a single argument tasks: an iterable of function aguments num_workers (None): a suggested number of threads to use - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar Returns: the list of function outputs @@ -841,7 +835,7 @@ def run(fcn, tasks, num_workers=None, progress=False): except: num_tasks = None - kwargs = dict(total=num_tasks, iters_str="files", quiet=not progress) + kwargs = dict(total=num_tasks, iters_str="files", progress=progress) if num_workers <= 1: with fou.ProgressBar(**kwargs) as pb: @@ -860,7 +854,7 @@ def _copy_files(inpaths, outpaths, skip_failures, progress): _run(_do_copy_file, tasks, progress=progress) -def _run(fcn, tasks, num_workers=None, progress=False): +def _run(fcn, tasks, num_workers=None, progress=None): num_workers = fou.recommend_thread_pool_workers(num_workers) try: @@ -868,7 +862,7 @@ def _run(fcn, tasks, num_workers=None, progress=False): except: num_tasks = None - kwargs = dict(total=num_tasks, iters_str="files", quiet=not progress) + kwargs = dict(total=num_tasks, iters_str="files", progress=progress) if num_workers <= 1: with fou.ProgressBar(**kwargs) as pb: diff --git a/fiftyone/core/utils.py b/fiftyone/core/utils.py index 58451c6be1..e2f0130c56 100644 --- a/fiftyone/core/utils.py +++ b/fiftyone/core/utils.py @@ -969,6 +969,7 @@ def __init__(self, total=None, progress=None, quiet=None, **kwargs): super().__init__(**kwargs) + self._progress = progress self._callback = callback def __call__(self, iterable): @@ -1108,6 +1109,7 @@ def __init__( self._last_batch_size = None self._pb = None self._in_context = False + self._render_progress = bool(progress) self._last_offset = None self._num_samples = None self._manually_applied_backpressure = True @@ -1119,7 +1121,7 @@ def __enter__(self): def __exit__(self, *args): self._in_context = False - if self.progress: + if self._render_progress: if self._last_batch_size is not None: self._pb.update(count=self._last_batch_size) @@ -1132,23 +1134,20 @@ def __iter__(self): else: self._iter = iter(self.iterable) - if self.progress: + if self._render_progress: if self._in_context: total = self.total if total is None: - try: - total = len(self.iterable) - except: - pass + total = self.iterable - self._pb = ProgressBar(total=total) + self._pb = ProgressBar(total=total, progress=self.progress) self._pb.__enter__() else: logger.warning( "Batcher must be invoked as a context manager in order to " "print progress" ) - self.progress = False + self._render_progress = False return self @@ -1160,9 +1159,10 @@ def __next__(self): raise ValueError( "Backpressure value not registered for this batcher" ) + self._manually_applied_backpressure = False - if self.progress and self._last_batch_size is not None: + if self._render_progress and self._last_batch_size is not None: self._pb.update(count=self._last_batch_size) batch_size = self._compute_batch_size() diff --git a/fiftyone/utils/data/importers.py b/fiftyone/utils/data/importers.py index 912f7a141d..0e61401fcf 100644 --- a/fiftyone/utils/data/importers.py +++ b/fiftyone/utils/data/importers.py @@ -1982,8 +1982,6 @@ def _parse_frame(fd): fomi.migrate_dataset_if_necessary(name) dataset._reload(hard=True) - logger.info("Import complete") - return sample_ids @staticmethod diff --git a/fiftyone/utils/eval/classification.py b/fiftyone/utils/eval/classification.py index 57f73ea91f..03ac9b3e07 100644 --- a/fiftyone/utils/eval/classification.py +++ b/fiftyone/utils/eval/classification.py @@ -89,6 +89,9 @@ def evaluate_classifications( Returns: a :class:`ClassificationResults` """ + if progress is None: + progress = True + fov.validate_collection_label_fields( samples, (pred_field, gt_field), fol.Classification, same_type=True ) diff --git a/fiftyone/utils/eval/detection.py b/fiftyone/utils/eval/detection.py index 3932e12941..47c59530e5 100644 --- a/fiftyone/utils/eval/detection.py +++ b/fiftyone/utils/eval/detection.py @@ -139,6 +139,9 @@ def evaluate_detections( Returns: a :class:`DetectionResults` """ + if progress is None: + progress = True + fov.validate_collection_label_fields( samples, (pred_field, gt_field), diff --git a/fiftyone/utils/eval/regression.py b/fiftyone/utils/eval/regression.py index 61d6995afe..aaa4eee5bf 100644 --- a/fiftyone/utils/eval/regression.py +++ b/fiftyone/utils/eval/regression.py @@ -82,6 +82,9 @@ def evaluate_regressions( Returns: a :class:`RegressionResults` """ + if progress is None: + progress = True + fov.validate_collection_label_fields( samples, (pred_field, gt_field), fol.Regression, same_type=True ) diff --git a/fiftyone/utils/eval/segmentation.py b/fiftyone/utils/eval/segmentation.py index f441a39c36..8645c94934 100644 --- a/fiftyone/utils/eval/segmentation.py +++ b/fiftyone/utils/eval/segmentation.py @@ -94,6 +94,9 @@ def evaluate_segmentations( Returns: a :class:`SegmentationResults` """ + if progress is None: + progress = True + fov.validate_collection_label_fields( samples, (pred_field, gt_field), fol.Segmentation, same_type=True ) diff --git a/fiftyone/utils/labels.py b/fiftyone/utils/labels.py index ab711c380c..d39b969da8 100644 --- a/fiftyone/utils/labels.py +++ b/fiftyone/utils/labels.py @@ -621,6 +621,7 @@ def classifications_to_detections( field out_field: the name of the :class:`fiftyone.core.labels.Detections` field to populate + progress (None): whether to render a progress bar """ fov.validate_collection_label_fields( sample_collection, in_field, fol.Classifications diff --git a/fiftyone/zoo/datasets/__init__.py b/fiftyone/zoo/datasets/__init__.py index eac04b7415..9601aadd78 100644 --- a/fiftyone/zoo/datasets/__init__.py +++ b/fiftyone/zoo/datasets/__init__.py @@ -236,9 +236,9 @@ def load_zoo_dataset( dataset is to be downloaded cleanup (True): whether to cleanup any temporary files generated during download - progress (None): whether to show the progress bar of the import. - If None this uses the global setting, otherwise it overwrites - the setting for this method. + progress (None): whether to show the progress bar of the import. If + None this uses the global setting, otherwise it overwrites the + setting for this method **kwargs: optional arguments to pass to the :class:`fiftyone.utils.data.importers.DatasetImporter` constructor. If ``download_if_necessary == True``, then ``kwargs`` can also @@ -347,14 +347,21 @@ def load_zoo_dataset( dataset_type, dataset_dir=split_dir, **importer_kwargs ) dataset.add_importer( - dataset_importer, label_field=label_field, tags=[split] + dataset_importer, + label_field=label_field, + tags=[split], + progress=progress, ) else: logger.info("Loading '%s'", zoo_dataset.name) dataset_importer, _ = foud.build_dataset_importer( dataset_type, dataset_dir=dataset_dir, **importer_kwargs ) - dataset.add_importer(dataset_importer, label_field=label_field) + dataset.add_importer( + dataset_importer, + label_field=label_field, + progress=progress, + ) if info.classes is not None: dataset.default_classes = info.classes diff --git a/tests/unittests/utils_tests.py b/tests/unittests/utils_tests.py index 91cbb6cdc7..ac7cac3f2a 100644 --- a/tests/unittests/utils_tests.py +++ b/tests/unittests/utils_tests.py @@ -457,11 +457,11 @@ def test_load_dataset_nonexistent(self, mock_get_db_conn): class ProgressBarTests(unittest.TestCase): def _test_correct_value(self, progress, global_progress, quiet, expected): - fo.config.show_progress_bars = global_progress - with fou.ProgressBar([], progress=progress, quiet=quiet) as pb: - assert pb._progress == expected + with fou.SetAttributes(fo.config, show_progress_bars=global_progress): + with fou.ProgressBar([], progress=progress, quiet=quiet) as pb: + assert pb._progress == expected - def test_progress_None_uses_global(self): + def test_progress_none_uses_global(self): self._test_correct_value( progress=None, global_progress=True, quiet=None, expected=True ) From 79111c48897ddaeec31106104431a15bb292512c Mon Sep 17 00:00:00 2001 From: brimoor Date: Tue, 2 Jan 2024 10:34:24 -0500 Subject: [PATCH 07/12] standardizing progress usage --- fiftyone/core/collections.py | 48 ++++++--- fiftyone/core/dataset.py | 141 +++++++++++++++++++------- fiftyone/core/metadata.py | 4 +- fiftyone/core/models.py | 12 ++- fiftyone/core/odm/database.py | 8 +- fiftyone/core/storage.py | 24 +++-- fiftyone/core/view.py | 21 ++-- fiftyone/utils/annotations.py | 8 +- fiftyone/utils/cvat.py | 8 +- fiftyone/utils/data/exporters.py | 12 ++- fiftyone/utils/data/importers.py | 12 ++- fiftyone/utils/data/parsers.py | 16 ++- fiftyone/utils/eval/activitynet.py | 4 +- fiftyone/utils/eval/classification.py | 11 +- fiftyone/utils/eval/coco.py | 4 +- fiftyone/utils/eval/detection.py | 11 +- fiftyone/utils/eval/openimages.py | 4 +- fiftyone/utils/eval/regression.py | 11 +- fiftyone/utils/eval/segmentation.py | 11 +- fiftyone/utils/geojson.py | 4 +- fiftyone/utils/image.py | 8 +- fiftyone/utils/iou.py | 8 +- fiftyone/utils/labelbox.py | 16 ++- fiftyone/utils/labels.py | 36 +++++-- fiftyone/utils/scale.py | 8 +- fiftyone/utils/utils3d.py | 4 +- fiftyone/utils/video.py | 12 ++- fiftyone/zoo/datasets/__init__.py | 6 +- 28 files changed, 333 insertions(+), 139 deletions(-) diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py index 9c601d2fe9..68dae5003e 100644 --- a/fiftyone/core/collections.py +++ b/fiftyone/core/collections.py @@ -830,7 +830,9 @@ def iter_samples(self, progress=False, autosave=False, batch_size=None): """Returns an iterator over the samples in the collection. Args: - progress (False): whether to render a progress bar + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -854,7 +856,9 @@ def iter_groups( Args: group_slices (None): an optional subset of group slices to load - progress (False): whether to render a progress bar + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -2747,7 +2751,9 @@ def compute_metadata( raising an error if metadata cannot be computed for a sample warn_failures (False): whether to log a warning if metadata cannot be computed for a sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ fomt.compute_metadata( self, @@ -2818,7 +2824,9 @@ def apply_model( subdirectories in ``output_dir`` that match the shape of the input paths. The path is converted to an absolute path (if necessary) via :func:`fiftyone.core.storage.normalize_path` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation """ @@ -2884,7 +2892,9 @@ def compute_embeddings( raising an error if embeddings cannot be generated for a sample. Only applicable to :class:`fiftyone.core.models.Model` instances - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation @@ -2981,7 +2991,9 @@ def compute_patch_embeddings( applicable for Torch-based models skip_failures (True): whether to gracefully continue without raising an error if embeddings cannot be generated for a sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: one of the following: @@ -3061,7 +3073,9 @@ def evaluate_regressions( The supported values are ``fo.evaluation_config.regression_backends.keys()`` and the default is ``fo.evaluation_config.regression_default_backend`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.regression.RegressionEvaluationConfig` being used @@ -3136,7 +3150,9 @@ def evaluate_classifications( The supported values are ``fo.evaluation_config.classification_backends.keys()`` and the default is ``fo.evaluation_config.classification_default_backend`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.classification.ClassificationEvaluationConfig` being used @@ -3262,7 +3278,9 @@ def evaluate_detections( label (True) or allow matches between classes (False) dynamic (True): whether to declare the dynamic object-level attributes that are populated on the dataset's schema - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.detection.DetectionEvaluationConfig` being used @@ -3348,7 +3366,9 @@ class for the purposes of computing evaluation metrics like The supported values are ``fo.evaluation_config.segmentation_backends.keys()`` and the default is ``fo.evaluation_config.segmentation_default_backend`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.segmentation.SegmentationEvaluationConfig` being used @@ -8430,7 +8450,9 @@ def export( overwrite (False): whether to delete existing directories before performing the export (True) or to merge the export with existing files and directories (False) - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the dataset exporter's constructor. If you are exporting image patches, this can also contain keyword arguments for @@ -9079,7 +9101,9 @@ def to_dict( readable format with newlines and indentations. Only applicable to datasets that contain videos when a ``frame_labels_dir`` is provided - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a JSON dict diff --git a/fiftyone/core/dataset.py b/fiftyone/core/dataset.py index 113ce26ca5..5ded0c744b 100644 --- a/fiftyone/core/dataset.py +++ b/fiftyone/core/dataset.py @@ -2164,7 +2164,9 @@ def make_label(): sample.ground_truth.label = make_label() Args: - progress (False): whether to render a progress bar + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -2174,9 +2176,6 @@ def make_label(): Returns: an iterator over :class:`fiftyone.core.sample.Sample` instances """ - if progress is None: - progress = False - with contextlib.ExitStack() as exit_context: samples = self._iter_samples() @@ -2266,7 +2265,9 @@ def make_label(): Args: group_slices (None): an optional subset of group slices to load - progress (False): whether to render a progress bar + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -2280,9 +2281,6 @@ def make_label(): if self.media_type != fom.GROUP: raise ValueError("%s does not contain groups" % type(self)) - if progress is None: - progress = False - with contextlib.ExitStack() as exit_context: groups = self._iter_groups(group_slices=group_slices) @@ -2462,7 +2460,9 @@ def add_samples( document fields that are encountered validate (True): whether to validate that the fields of each sample are compliant with the dataset schema before adding it - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead num_samples (None): the number of samples in ``samples``. If not provided, this is computed (if possible) via ``len(samples)`` if needed for progress tracking @@ -2512,7 +2512,9 @@ def add_collection( information. Only applicable when ``include_info`` is True new_ids (False): whether to generate new sample/frame/group IDs. By default, the IDs of the input collection are retained - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to this dataset @@ -2900,7 +2902,9 @@ def merge_samples( information. Only applicable when ``samples`` is a :class:`fiftyone.core.collections.SampleCollection` and ``include_info`` is True - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead num_samples (None): the number of samples in ``samples``. If not provided, this is computed (if possibleE) via ``len(samples)`` if needed for progress tracking @@ -4059,7 +4063,9 @@ def add_dir( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset's ``info`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4263,7 +4269,9 @@ def merge_dir( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4392,7 +4400,9 @@ def add_archive( add_info (True): whether to add dataset info from the importer (if any) to the dataset's ``info`` cleanup (True): whether to delete the archive after extracting it - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4589,7 +4599,9 @@ def merge_archive( add_info (True): whether to add dataset info from the importer (if any) to the dataset cleanup (True): whether to delete the archive after extracting it - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4659,7 +4671,9 @@ def add_importer( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset's ``info`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -4795,7 +4809,9 @@ def merge_importer( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ return foud.merge_samples( self, @@ -4841,7 +4857,9 @@ def add_images( instance to use to parse the samples tags (None): an optional tag or iterable of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -4898,7 +4916,9 @@ def add_labeled_images( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -4933,7 +4953,9 @@ def add_images_dir( tags (None): an optional tag or iterable of tags to attach to each sample recursive (True): whether to recursively traverse subdirectories - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -4954,7 +4976,9 @@ def add_images_patt(self, images_patt, tags=None, progress=None): ``/path/to/images/*.jpg`` tags (None): an optional tag or iterable of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -4996,6 +5020,9 @@ def ingest_images( written. By default, :func:`get_default_dataset_dir` is used image_format (None): the image format to use to write the images to disk. By default, ``fiftyone.config.default_image_ext`` is used + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -5063,7 +5090,9 @@ def ingest_labeled_images( written. By default, :func:`get_default_dataset_dir` is used image_format (None): the image format to use to write the images to disk. By default, ``fiftyone.config.default_image_ext`` is used - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -5112,7 +5141,9 @@ def add_videos( instance to use to parse the samples tags (None): an optional tag or iterable of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -5167,7 +5198,9 @@ def add_labeled_videos( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -5198,7 +5231,9 @@ def add_videos_dir( tags (None): an optional tag or iterable of tags to attach to each sample recursive (True): whether to recursively traverse subdirectories - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -5219,7 +5254,9 @@ def add_videos_patt(self, videos_patt, tags=None, progress=None): ``/path/to/videos/*.mp4`` tags (None): an optional tag or iterable of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -5258,7 +5295,9 @@ def ingest_videos( sample dataset_dir (None): the directory in which the videos will be written. By default, :func:`get_default_dataset_dir` is used - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -5310,7 +5349,9 @@ def ingest_labeled_videos( document fields that are encountered dataset_dir (None): the directory in which the videos will be written. By default, :func:`get_default_dataset_dir` is used - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -5432,7 +5473,9 @@ def from_dir( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -5554,7 +5597,9 @@ def from_archive( dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered cleanup (True): whether to delete the archive after extracting it - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -5623,7 +5668,9 @@ def from_importer( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5674,7 +5721,9 @@ def from_images( the same name tags (None): an optional tag or iterable of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5735,7 +5784,9 @@ def from_labeled_images( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5777,7 +5828,9 @@ def from_images_dir( tags (None): an optional tag or iterable of tags to attach to each sample recursive (True): whether to recursively traverse subdirectories - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5813,7 +5866,9 @@ def from_images_patt( the same name tags (None): an optional tag or iterable of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5858,7 +5913,9 @@ def from_videos( the same name tags (None): an optional tag or iterable of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5920,7 +5977,9 @@ def from_labeled_videos( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -6039,7 +6098,9 @@ def from_dict( is assumed that the frame labels are included directly in the provided JSON dict. Only applicable to datasets that contain videos - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -6157,7 +6218,9 @@ def from_json( of each sample, if the filepath is not absolute (begins with a path separator). The path is converted to an absolute path (if necessary) via :func:`fiftyone.core.storage.normalize_path` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` diff --git a/fiftyone/core/metadata.py b/fiftyone/core/metadata.py index bbb714bff1..605ef1d59e 100644 --- a/fiftyone/core/metadata.py +++ b/fiftyone/core/metadata.py @@ -251,7 +251,9 @@ def compute_metadata( error if metadata cannot be computed for a sample warn_failures (False): whether to log a warning if metadata cannot be computed for a sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ num_workers = fou.recommend_thread_pool_workers(num_workers) diff --git a/fiftyone/core/models.py b/fiftyone/core/models.py index 525212eb0d..bd542fe1ac 100644 --- a/fiftyone/core/models.py +++ b/fiftyone/core/models.py @@ -100,7 +100,9 @@ def apply_model( ``output_dir`` that match the shape of the input paths. The path is converted to an absolute path (if necessary) via :func:`fiftyone.core.storage.normalize_path` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation """ @@ -764,7 +766,9 @@ def compute_embeddings( skip_failures (True): whether to gracefully continue without raising an error if embeddings cannot be generated for a sample. Only applicable to :class:`Model` instances - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation @@ -1282,7 +1286,9 @@ def compute_patch_embeddings( Only applicable for Torch models skip_failures (True): whether to gracefully continue without raising an error if embeddings cannot be generated for a sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: one of the following: diff --git a/fiftyone/core/odm/database.py b/fiftyone/core/odm/database.py index 7d5459b3bf..38c6d81501 100644 --- a/fiftyone/core/odm/database.py +++ b/fiftyone/core/odm/database.py @@ -648,7 +648,9 @@ def export_collection( to the document's ID num_docs (None): the total number of documents. If omitted, this must be computable via ``len(docs)`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if num_docs is None: num_docs = len(docs) @@ -755,7 +757,9 @@ def insert_documents(docs, coll, ordered=False, progress=None, num_docs=None): docs: an iterable of BSON document dicts coll: a pymongo collection ordered (False): whether the documents must be inserted in order - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead num_docs (None): the total number of documents. Only used when ``progress=True``. If omitted, this will be computed via ``len(docs)``, if possible diff --git a/fiftyone/core/storage.py b/fiftyone/core/storage.py index 7d7978b23d..0b05a94d9c 100644 --- a/fiftyone/core/storage.py +++ b/fiftyone/core/storage.py @@ -697,7 +697,9 @@ def copy_files(inpaths, outpaths, skip_failures=False, progress=None): outpaths: a list of output paths skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ _copy_files(inpaths, outpaths, skip_failures, progress) @@ -714,7 +716,9 @@ def copy_dir( or merge its contents (False) skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if overwrite and os.path.isdir(outdir): delete_dir(outdir) @@ -747,7 +751,9 @@ def move_files(inpaths, outpaths, skip_failures=False, progress=None): outpaths: a list of output paths skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ tasks = [(i, o, skip_failures) for i, o in zip(inpaths, outpaths)] if tasks: @@ -767,7 +773,9 @@ def move_dir( or merge its contents (False) skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if overwrite and os.path.isdir(outdir): delete_dir(outdir) @@ -799,7 +807,9 @@ def delete_files(paths, skip_failures=False, progress=None): paths: a list of paths skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ tasks = [(p, skip_failures) for p in paths] if tasks: @@ -823,7 +833,9 @@ def run(fcn, tasks, num_workers=None, progress=None): fcn: a function that accepts a single argument tasks: an iterable of function aguments num_workers (None): a suggested number of threads to use - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: the list of function outputs diff --git a/fiftyone/core/view.py b/fiftyone/core/view.py index 7be74d72e3..ca52b9a10f 100644 --- a/fiftyone/core/view.py +++ b/fiftyone/core/view.py @@ -466,7 +466,9 @@ def make_label(): sample.ground_truth.label = make_label() Args: - progress (False): whether to render a progress bar + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -476,9 +478,6 @@ def make_label(): Returns: an iterator over :class:`fiftyone.core.sample.SampleView` instances """ - if progress is None: - progress = False - with contextlib.ExitStack() as exit_context: samples = self._iter_samples() @@ -583,7 +582,9 @@ def make_label(): Args: group_slices (None): an optional subset of group slices to load - progress (False): whether to render a progress bar + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -602,9 +603,6 @@ def make_label(): "Use iter_dynamic_groups() for dynamic group views" ) - if progress is None: - progress = False - with contextlib.ExitStack() as exit_context: groups = self._iter_groups(group_slices=group_slices) @@ -681,7 +679,9 @@ def iter_dynamic_groups(self, progress=False): print("%s: %d" % (group_value, len(group))) Args: - progress (False): whether to render a progress bar + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: an iterator that emits :class:`DatasetView` instances, one per @@ -690,9 +690,6 @@ def iter_dynamic_groups(self, progress=False): if not self._is_dynamic_groups: raise ValueError("%s does not contain dynamic groups" % type(self)) - if progress is None: - progress = False - with contextlib.ExitStack() as context: groups = self._iter_dynamic_groups() diff --git a/fiftyone/utils/annotations.py b/fiftyone/utils/annotations.py index 7b3a55ec8c..f8344b288d 100644 --- a/fiftyone/utils/annotations.py +++ b/fiftyone/utils/annotations.py @@ -1020,7 +1020,9 @@ def load_annotations( or ``None`` if there aren't any cleanup (False): whether to delete any informtation regarding this run from the annotation backend after loading the annotations - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: keyword arguments for the run's :meth:`fiftyone.core.annotation.AnnotationMethodConfig.load_credentials` method @@ -2343,7 +2345,9 @@ def draw_labeled_images( If omitted, all compatiable fields are rendered config (None): an optional :class:`DrawConfig` configuring how to draw the labels - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments specifying parameters of the default :class:`DrawConfig` to override diff --git a/fiftyone/utils/cvat.py b/fiftyone/utils/cvat.py index 78fdfcf770..20fbe2c23b 100644 --- a/fiftyone/utils/cvat.py +++ b/fiftyone/utils/cvat.py @@ -4000,7 +4000,9 @@ def delete_projects(self, project_ids, progress=None): Args: project_ids: an iterable of project IDs - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ with fou.ProgressBar(progress=progress) as pb: for project_id in pb(list(project_ids)): @@ -4158,7 +4160,9 @@ def delete_tasks(self, task_ids, progress=None): Args: task_ids: an iterable of task IDs - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ with fou.ProgressBar(progress=progress) as pb: for task_id in pb(list(task_ids)): diff --git a/fiftyone/utils/data/exporters.py b/fiftyone/utils/data/exporters.py index cc35d2e166..86a015389d 100644 --- a/fiftyone/utils/data/exporters.py +++ b/fiftyone/utils/data/exporters.py @@ -193,7 +193,9 @@ def export_samples( or a dictionary mapping field names to output keys describing the frame label fields to export. Only applicable if ``dataset_exporter`` is a :class:`LabeledVideoDatasetExporter` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead num_samples (None): the number of samples in ``samples``. If omitted, this is computed (if possible) via ``len(samples)`` if needed for progress tracking @@ -389,7 +391,9 @@ def write_dataset( :class:`fiftyone.core.collections.SampleCollection`, this parameter defaults to ``samples``. This parameter is optional and is only passed to :meth:`DatasetExporter.log_collection` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead num_samples (None): the number of samples in ``samples``. If omitted, this is computed (if possible) via ``len(samples)`` if needed for progress tracking @@ -1397,7 +1401,9 @@ def export_samples(self, sample_collection, progress=None): Args: sample_collection: a :class:`fiftyone.core.collections.SampleCollection` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ raise NotImplementedError("subclass must implement export_samples()") diff --git a/fiftyone/utils/data/importers.py b/fiftyone/utils/data/importers.py index 0e61401fcf..1ebeb2c603 100644 --- a/fiftyone/utils/data/importers.py +++ b/fiftyone/utils/data/importers.py @@ -88,7 +88,9 @@ def import_samples( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -266,7 +268,9 @@ def merge_samples( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if etau.is_str(tags): tags = [tags] @@ -988,7 +992,9 @@ def import_samples(self, dataset, tags=None, progress=None): Args: dataset: a :class:`fiftyone.core.dataset.Dataset` tags (None): an optional list of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset diff --git a/fiftyone/utils/data/parsers.py b/fiftyone/utils/data/parsers.py index 20bbf0a5f5..220f92de7e 100644 --- a/fiftyone/utils/data/parsers.py +++ b/fiftyone/utils/data/parsers.py @@ -40,7 +40,9 @@ def add_images(dataset, samples, sample_parser, tags=None, progress=None): parse the samples tags (None): an optional tag or iterable of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -127,7 +129,9 @@ def add_labeled_images( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -218,7 +222,9 @@ def add_videos(dataset, samples, sample_parser, tags=None, progress=None): parse the samples tags (None): an optional tag or iterable of tags to attach to each sample - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -299,7 +305,9 @@ def add_labeled_videos( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset diff --git a/fiftyone/utils/eval/activitynet.py b/fiftyone/utils/eval/activitynet.py index d15fa07ee6..0965352065 100644 --- a/fiftyone/utils/eval/activitynet.py +++ b/fiftyone/utils/eval/activitynet.py @@ -152,7 +152,9 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched segments are given this label for results purposes - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`DetectionResults` diff --git a/fiftyone/utils/eval/classification.py b/fiftyone/utils/eval/classification.py index 03ac9b3e07..5b1d8e360d 100644 --- a/fiftyone/utils/eval/classification.py +++ b/fiftyone/utils/eval/classification.py @@ -82,16 +82,15 @@ def evaluate_classifications( supported values are ``fo.evaluation_config.classification_backends.keys()`` and the default is ``fo.evaluation_config.default_classification_backend`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`ClassificationEvaluationConfig` being used Returns: a :class:`ClassificationResults` """ - if progress is None: - progress = True - fov.validate_collection_label_fields( samples, (pred_field, gt_field), fol.Classification, same_type=True ) @@ -170,7 +169,9 @@ def evaluate_samples( purposes missing (None): a missing label string. Any None-valued labels are given this label for results purposes - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`ClassificationResults` instance diff --git a/fiftyone/utils/eval/coco.py b/fiftyone/utils/eval/coco.py index cd9a2508b1..3efe7838d4 100644 --- a/fiftyone/utils/eval/coco.py +++ b/fiftyone/utils/eval/coco.py @@ -194,7 +194,9 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched objects are given this label for results purposes - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`DetectionResults` diff --git a/fiftyone/utils/eval/detection.py b/fiftyone/utils/eval/detection.py index 47c59530e5..5bf8b9081f 100644 --- a/fiftyone/utils/eval/detection.py +++ b/fiftyone/utils/eval/detection.py @@ -132,16 +132,15 @@ def evaluate_detections( label (True) or allow matches between classes (False) dynamic (True): whether to declare the dynamic object-level attributes that are populated on the dataset's schema - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`DetectionEvaluationConfig` being used Returns: a :class:`DetectionResults` """ - if progress is None: - progress = True - fov.validate_collection_label_fields( samples, (pred_field, gt_field), @@ -394,7 +393,9 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched objects are given this label for results purposes - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`DetectionResults` diff --git a/fiftyone/utils/eval/openimages.py b/fiftyone/utils/eval/openimages.py index 65599ef938..d92d371996 100644 --- a/fiftyone/utils/eval/openimages.py +++ b/fiftyone/utils/eval/openimages.py @@ -244,7 +244,9 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched objects are given this label for results purposes - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`OpenImagesDetectionResults` diff --git a/fiftyone/utils/eval/regression.py b/fiftyone/utils/eval/regression.py index aaa4eee5bf..1d1fc17872 100644 --- a/fiftyone/utils/eval/regression.py +++ b/fiftyone/utils/eval/regression.py @@ -75,16 +75,15 @@ def evaluate_regressions( supported values are ``fo.evaluation_config.regression_backends.keys()`` and the default is ``fo.evaluation_config.default_regression_backend`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`RegressionEvaluationConfig` being used Returns: a :class:`RegressionResults` """ - if progress is None: - progress = True - fov.validate_collection_label_fields( samples, (pred_field, gt_field), fol.Regression, same_type=True ) @@ -165,7 +164,9 @@ def evaluate_samples( eval_key (None): an evaluation key for this evaluation missing (None): a missing value. Any None-valued regressions are given this value for results purposes - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`RegressionResults` instance diff --git a/fiftyone/utils/eval/segmentation.py b/fiftyone/utils/eval/segmentation.py index 8645c94934..68ff2f8170 100644 --- a/fiftyone/utils/eval/segmentation.py +++ b/fiftyone/utils/eval/segmentation.py @@ -87,16 +87,15 @@ def evaluate_segmentations( supported values are ``fo.evaluation_config.segmentation_backends.keys()`` and the default is ``fo.evaluation_config.default_segmentation_backend`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`SegmentationEvaluationConfig` being used Returns: a :class:`SegmentationResults` """ - if progress is None: - progress = True - fov.validate_collection_label_fields( samples, (pred_field, gt_field), fol.Segmentation, same_type=True ) @@ -198,7 +197,9 @@ def evaluate_samples( contain a subset of the possible classes if you wish to evaluate a subset of the semantic classes. By default, the observed pixel values are used as labels - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`SegmentationResults` instance diff --git a/fiftyone/utils/geojson.py b/fiftyone/utils/geojson.py index 6391c1968b..b5e53f2264 100644 --- a/fiftyone/utils/geojson.py +++ b/fiftyone/utils/geojson.py @@ -90,7 +90,9 @@ def load_location_data( used, else a new "location" field is created skip_missing (True): whether to skip GeoJSON features with no ``filename`` properties (True) or raise an error (False) - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if location_field is None: try: diff --git a/fiftyone/utils/image.py b/fiftyone/utils/image.py index aad5a947e6..7701569067 100644 --- a/fiftyone/utils/image.py +++ b/fiftyone/utils/image.py @@ -101,7 +101,9 @@ def reencode_images( num_workers (None): a suggested number of worker processes to use skip_failures (False): whether to gracefully continue without raising an error if an image cannot be re-encoded - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_image_collection(sample_collection) @@ -193,7 +195,9 @@ def transform_images( num_workers (None): a suggested number of worker processes to use skip_failures (False): whether to gracefully continue without raising an error if an image cannot be transformed - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_image_collection(sample_collection) diff --git a/fiftyone/utils/iou.py b/fiftyone/utils/iou.py index fff7c1a584..bc6e1e4ef8 100644 --- a/fiftyone/utils/iou.py +++ b/fiftyone/utils/iou.py @@ -176,7 +176,9 @@ def compute_max_ious( iou_attr ("max_iou"): the label attribute in which to store the max IoU id_attr (None): an optional attribute in which to store the label ID of the maximum overlapping label - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for :func:`compute_ious` """ if other_field is None: @@ -294,7 +296,9 @@ def find_duplicates( labels are duplicates method ("simple"): the duplicate removal method to use. The supported values are ``("simple", "greedy")`` - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for :func:`compute_ious` Returns: diff --git a/fiftyone/utils/labelbox.py b/fiftyone/utils/labelbox.py index 0661be48d6..fe0d527998 100644 --- a/fiftyone/utils/labelbox.py +++ b/fiftyone/utils/labelbox.py @@ -402,7 +402,9 @@ def delete_datasets(self, dataset_ids, progress=None): Args: dataset_ids: an iterable of dataset IDs - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ logger.info("Deleting datasets...") with fou.ProgressBar(progress=progress) as pb: @@ -1548,7 +1550,9 @@ def import_from_labelbox( samples labelbox_id_field ("labelbox_id"): the sample field to lookup/store the IDs of the Labelbox DataRows - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection(dataset, media_type=(fomm.IMAGE, fomm.VIDEO)) is_video = dataset.media_type == fomm.VIDEO @@ -1694,7 +1698,9 @@ def export_to_labelbox( when constructing the exported frame labels By default, no frame labels are exported - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection( sample_collection, media_type=(fomm.IMAGE, fomm.VIDEO) @@ -1814,7 +1820,9 @@ def upload_media_to_labelbox( :class:`fiftyone.core.collections.SampleCollection` labelbox_id_field ("labelbox_id"): the sample field in which to store the IDs of the Labelbox DataRows - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ # @todo use `create_data_rows()` to optimize performance # @todo handle API rate limits diff --git a/fiftyone/utils/labels.py b/fiftyone/utils/labels.py index d39b969da8..1bb8caa3cd 100644 --- a/fiftyone/utils/labels.py +++ b/fiftyone/utils/labels.py @@ -62,7 +62,9 @@ def objects_to_segmentations( if it exists save_mask_targets (False): whether to store the ``mask_targets`` on the dataset - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, @@ -173,7 +175,9 @@ def export_segmentations( update (True): whether to delete the arrays from the database overwrite (False): whether to delete ``output_dir`` prior to exporting if it exists - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, (fol.Segmentation, fol.Heatmap) @@ -232,7 +236,9 @@ def import_segmentations( :class:`fiftyone.core.labels.Heatmap` field update (True): whether to delete the image paths from the labels delete_images (False): whether to delete any imported images from disk - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, (fol.Segmentation, fol.Heatmap) @@ -309,7 +315,9 @@ def transform_segmentations( dataset to reflect the transformed targets overwrite (False): whether to delete ``output_dir`` prior to exporting if it exists - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, fol.Segmentation @@ -410,7 +418,9 @@ def segmentations_to_detections( - ``"thing"`` if all classes are thing classes - a dict mapping pixel values (2D masks) or RGB hex strings (3D masks) to ``"stuff"`` or ``"thing"`` for each class - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, @@ -463,7 +473,9 @@ def instances_to_polylines( tolerance (2): a tolerance, in pixels, when generating approximate polylines for each region. Typical values are 1-3 pixels filled (True): whether the polylines should be filled - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, @@ -533,7 +545,9 @@ def segmentations_to_polylines( masks) to ``"stuff"`` or ``"thing"`` for each class tolerance (2): a tolerance, in pixels, when generating approximate polylines for each region. Typical values are 1-3 pixels - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, @@ -577,7 +591,9 @@ def classification_to_detections( field out_field: the name of the :class:`fiftyone.core.labels.Detections` field to populate - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, fol.Classification @@ -621,7 +637,9 @@ def classifications_to_detections( field out_field: the name of the :class:`fiftyone.core.labels.Detections` field to populate - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, fol.Classifications diff --git a/fiftyone/utils/scale.py b/fiftyone/utils/scale.py index 32f76bc0d2..d594a0bda5 100644 --- a/fiftyone/utils/scale.py +++ b/fiftyone/utils/scale.py @@ -187,7 +187,9 @@ def import_from_scale( that are created, separated by an underscore scale_id_field ("scale_id"): the sample field to use to associate Scale task IDs with FiftyOne samples - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection(dataset, media_type=(fomm.IMAGE, fomm.VIDEO)) is_video = dataset.media_type == fomm.VIDEO @@ -396,7 +398,9 @@ def export_to_scale( when constructing the exported frame labels By default, no frame labels are exported - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection( sample_collection, media_type=(fomm.IMAGE, fomm.VIDEO) diff --git a/fiftyone/utils/utils3d.py b/fiftyone/utils/utils3d.py index 00339acf92..501ee77c2e 100644 --- a/fiftyone/utils/utils3d.py +++ b/fiftyone/utils/utils3d.py @@ -510,7 +510,9 @@ def compute_orthographic_projection_images( to generate each map. Either element of the tuple or any/all of its values can be None, in which case a tight crop of the point cloud along the missing dimension(s) are used - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if in_group_slice is None and samples.media_type == fom.GROUP: in_group_slice = _get_point_cloud_slice(samples) diff --git a/fiftyone/utils/video.py b/fiftyone/utils/video.py index 38c1700b66..bbeda4af62 100644 --- a/fiftyone/utils/video.py +++ b/fiftyone/utils/video.py @@ -153,7 +153,9 @@ def reencode_videos( an error if a video cannot be re-encoded verbose (False): whether to log the ``ffmpeg`` commands that are executed - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: keyword arguments for ``eta.core.video.FFmpeg(**kwargs)`` """ fov.validate_video_collection(sample_collection) @@ -262,7 +264,9 @@ def transform_videos( an error if a video cannot be transformed verbose (False): whether to log the ``ffmpeg`` commands that are executed - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: keyword arguments for ``eta.core.video.FFmpeg(**kwargs)`` """ fov.validate_video_collection(sample_collection) @@ -411,7 +415,9 @@ def sample_videos( an error if a video cannot be sampled verbose (False): whether to log the ``ffmpeg`` commands that are executed - progress (None): whether to render a progress bar + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: keyword arguments for ``eta.core.video.FFmpeg(**kwargs)`` """ fov.validate_video_collection(sample_collection) diff --git a/fiftyone/zoo/datasets/__init__.py b/fiftyone/zoo/datasets/__init__.py index 9601aadd78..3b00e87ba1 100644 --- a/fiftyone/zoo/datasets/__init__.py +++ b/fiftyone/zoo/datasets/__init__.py @@ -236,9 +236,9 @@ def load_zoo_dataset( dataset is to be downloaded cleanup (True): whether to cleanup any temporary files generated during download - progress (None): whether to show the progress bar of the import. If - None this uses the global setting, otherwise it overwrites the - setting for this method + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional arguments to pass to the :class:`fiftyone.utils.data.importers.DatasetImporter` constructor. If ``download_if_necessary == True``, then ``kwargs`` can also From d0b7cb19ad89fd131b111bc481b85a9520e17897 Mon Sep 17 00:00:00 2001 From: brimoor Date: Tue, 2 Jan 2024 11:01:25 -0500 Subject: [PATCH 08/12] no progress by default for batchers --- fiftyone/core/dataset.py | 2 +- fiftyone/core/utils.py | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/fiftyone/core/dataset.py b/fiftyone/core/dataset.py index 5ded0c744b..3610da7946 100644 --- a/fiftyone/core/dataset.py +++ b/fiftyone/core/dataset.py @@ -2906,7 +2906,7 @@ def merge_samples( the default value ``fiftyone.config.show_progress_bars`` (None), or a progress callback function to invoke instead num_samples (None): the number of samples in ``samples``. If not - provided, this is computed (if possibleE) via ``len(samples)`` + provided, this is computed (if possible) via ``len(samples)`` if needed for progress tracking """ if fields is not None: diff --git a/fiftyone/core/utils.py b/fiftyone/core/utils.py index e2f0130c56..92c70f8644 100644 --- a/fiftyone/core/utils.py +++ b/fiftyone/core/utils.py @@ -1089,7 +1089,7 @@ def __init__( self, iterable, return_views=False, - progress=None, + progress=False, total=None, ): import fiftyone.core.collections as foc @@ -1109,7 +1109,7 @@ def __init__( self._last_batch_size = None self._pb = None self._in_context = False - self._render_progress = bool(progress) + self._render_progress = bool(progress) # callback function: True self._last_offset = None self._num_samples = None self._manually_applied_backpressure = True @@ -1325,7 +1325,9 @@ class LatencyDynamicBatcher(BaseDynamicBatcher): :class:`fiftyone.core.view.DatasetView`. Only applicable when the iterable is a :class:`fiftyone.core.collections.SampleCollection` progress (False): whether to render a progress bar tracking the - consumption of the batches + consumption of the batches (True/False), use the default value + ``fiftyone.config.show_progress_bars`` (None), or a progress + callback function to invoke instead total (None): the length of ``iterable``. Only applicable when ``progress=True``. If not provided, it is computed via ``len(iterable)``, if possible @@ -1430,7 +1432,9 @@ class BSONSizeDynamicBatcher(BaseDynamicBatcher): :class:`fiftyone.core.view.DatasetView`. Only applicable when the iterable is a :class:`fiftyone.core.collections.SampleCollection` progress (False): whether to render a progress bar tracking the - consumption of the batches + consumption of the batches (True/False), use the default value + ``fiftyone.config.show_progress_bars`` (None), or a progress + callback function to invoke instead total (None): the length of ``iterable``. Only applicable when ``progress=True``. If not provided, it is computed via ``len(iterable)``, if possible @@ -1505,7 +1509,9 @@ class StaticBatcher(Batcher): :class:`fiftyone.core.view.DatasetView`. Only applicable when the iterable is a :class:`fiftyone.core.collections.SampleCollection` progress (False): whether to render a progress bar tracking the - consumption of the batches + consumption of the batches (True/False), use the default value + ``fiftyone.config.show_progress_bars`` (None), or a progress + callback function to invoke instead total (None): the length of ``iterable``. Only applicable when ``progress=True``. If not provided, it is computed via ``len(iterable)``, if possible @@ -1529,7 +1535,7 @@ def _compute_batch_size(self): return self.batch_size -def get_default_batcher(iterable, progress=True, total=None): +def get_default_batcher(iterable, progress=False, total=None): """Returns a :class:`Batcher` over ``iterable`` using defaults from your FiftyOne config. @@ -1538,8 +1544,10 @@ def get_default_batcher(iterable, progress=True, total=None): Args: iterable: an iterable to batch over - progress (True): whether to render a progress bar tracking the - consumption of the batches + progress (False): whether to render a progress bar tracking the + consumption of the batches (True/False), use the default value + ``fiftyone.config.show_progress_bars`` (None), or a progress + callback function to invoke instead total (None): the length of ``iterable``. Only applicable when ``progress=True``. If not provided, it is computed via ``len(iterable)``, if possible From a34fd05b159adebb8d5b2d99a8270c86ff16629e Mon Sep 17 00:00:00 2001 From: brimoor Date: Tue, 2 Jan 2024 11:59:18 -0500 Subject: [PATCH 09/12] documenting progress callback --- docs/source/plugins/developing_plugins.rst | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index 80f747436b..a3cedeea42 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -1145,6 +1145,7 @@ on their execution context from within .. code-block:: python :linenos: + import fiftyone as fo import fiftyone.core.storage as fos import fiftyone.core.utils as fou @@ -1168,6 +1169,35 @@ on their execution context from within of their delegated operations from the :ref:`Runs page ` of the Teams App! +For your convenience, all builtin methods of the FiftyOne SDK that support +rendering progress bars provide an optional `progress` method that you can use +trigger calls to +:meth:`set_progress() ` +using the pattern show below: + +.. code-block:: python + :linenos: + + import fiftyone as fo + + def execute(self, ctx): + images_dir = ctx.params["images_dir"] + + # Custom logic that controls how progress is reported + def set_progress(pb): + if pb.complete: + ctx.set_progress(progress=1, label="Operation complete") + else: + ctx.set_progress(progress=pb.progress) + + # Option 1: report progress every five seconds + progress = fo.report_progress(set_progress, dt=5.0) + + # Option 2: report progress at 10 equally-spaced increments + # progress = fo.report_progress(set_progress, n=10) + + ctx.dataset.add_images_dir(images_dir, progress=progress) + .. _operator-execution: Operator execution From 23a0d1db95af5de33f8f4a611f95f1dcd8a67c3c Mon Sep 17 00:00:00 2001 From: brimoor Date: Tue, 2 Jan 2024 14:12:23 -0500 Subject: [PATCH 10/12] adding progress flag to draw_labels() --- fiftyone/core/collections.py | 6 ++++++ fiftyone/utils/annotations.py | 13 +++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py index 68dae5003e..19a1d14eb9 100644 --- a/fiftyone/core/collections.py +++ b/fiftyone/core/collections.py @@ -8188,6 +8188,7 @@ def draw_labels( label_fields=None, overwrite=False, config=None, + progress=None, **kwargs, ): """Renders annotated versions of the media in the collection with the @@ -8217,6 +8218,9 @@ def draw_labels( config (None): an optional :class:`fiftyone.utils.annotations.DrawConfig` configuring how to draw the labels + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments specifying parameters of the default :class:`fiftyone.utils.annotations.DrawConfig` to override @@ -8244,6 +8248,7 @@ def draw_labels( rel_dir=rel_dir, label_fields=label_fields, config=config, + progress=progress, **kwargs, ) @@ -8254,6 +8259,7 @@ def draw_labels( rel_dir=rel_dir, label_fields=label_fields, config=config, + progress=progress, **kwargs, ) diff --git a/fiftyone/utils/annotations.py b/fiftyone/utils/annotations.py index f8344b288d..66c5c9a0e2 100644 --- a/fiftyone/utils/annotations.py +++ b/fiftyone/utils/annotations.py @@ -2404,7 +2404,13 @@ def draw_labeled_image( def draw_labeled_videos( - samples, output_dir, rel_dir=None, label_fields=None, config=None, **kwargs + samples, + output_dir, + rel_dir=None, + label_fields=None, + config=None, + progress=None, + **kwargs, ): """Renders annotated versions of the videos in the collection with the specified label data overlaid to the given directory. @@ -2429,6 +2435,9 @@ def draw_labeled_videos( If omitted, all compatiable fields are rendered config (None): an optional :class:`DrawConfig` configuring how to draw the labels + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments specifying parameters of the default :class:`DrawConfig` to override @@ -2448,7 +2457,7 @@ def draw_labeled_videos( num_videos = len(samples) outpaths = [] - for idx, sample in enumerate(samples, 1): + for idx, sample in enumerate(samples.iter_samples(progress=progress), 1): if is_clips: logger.info("Drawing labels for clip %d/%d", idx, num_videos) base, ext = os.path.splitext(sample.filepath) From 115198e5d2ff9d187d2ee4f39dc1f1a9fbe36c16 Mon Sep 17 00:00:00 2001 From: brimoor Date: Tue, 2 Jan 2024 18:12:37 -0500 Subject: [PATCH 11/12] adding progress flag to load_annotations() --- fiftyone/core/collections.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py index 19a1d14eb9..243ec4f268 100644 --- a/fiftyone/core/collections.py +++ b/fiftyone/core/collections.py @@ -8752,6 +8752,7 @@ def load_annotations( dest_field=None, unexpected="prompt", cleanup=False, + progress=None, **kwargs, ): """Downloads the labels from the given annotation run from the @@ -8779,6 +8780,9 @@ def load_annotations( labels, or ``None`` if there aren't any cleanup (False): whether to delete any informtation regarding this run from the annotation backend after loading the annotations + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: keyword arguments for the run's :meth:`fiftyone.core.annotation.AnnotationMethodConfig.load_credentials` method @@ -8793,6 +8797,7 @@ def load_annotations( dest_field=dest_field, unexpected=unexpected, cleanup=cleanup, + progress=progress, **kwargs, ) From e53abdcf8dc2cc49c3c4c231fb55d82ef8001c8a Mon Sep 17 00:00:00 2001 From: brimoor Date: Tue, 2 Jan 2024 22:02:12 -0500 Subject: [PATCH 12/12] adding a ProgressHandler --- docs/source/plugins/developing_plugins.rst | 20 +++++++++ fiftyone/operators/__init__.py | 1 + fiftyone/operators/utils.py | 50 ++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 fiftyone/operators/utils.py diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index a3cedeea42..9ee8eb1561 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -1198,6 +1198,26 @@ using the pattern show below: ctx.dataset.add_images_dir(images_dir, progress=progress) +You can also use the builtin +:class:`ProgressHandler ` class to +automatically forward logging messages to +:meth:`set_progress() ` +as `label` values using the pattern shown below: + +.. code-block:: python + :linenos: + + import logging + import fiftyone.operators as foo + import fiftyone.zoo as foz + + def execute(self, ctx): + name = ctx.params["name"] + + # Automatically report all `fiftyone` logging messages + with foo.ProgressHandler(ctx, logger=logging.getLogger("fiftyone")): + foz.load_zoo_dataset(name, persistent=True) + .. _operator-execution: Operator execution diff --git a/fiftyone/operators/__init__.py b/fiftyone/operators/__init__.py index d90e6d3c86..6d389bb4ab 100644 --- a/fiftyone/operators/__init__.py +++ b/fiftyone/operators/__init__.py @@ -16,6 +16,7 @@ execute_operator, ExecutionOptions, ) +from .utils import ProgressHandler # This enables Sphinx refs to directly use paths imported here __all__ = [k for k, v in globals().items() if not k.startswith("_")] diff --git a/fiftyone/operators/utils.py b/fiftyone/operators/utils.py new file mode 100644 index 0000000000..337358d2a8 --- /dev/null +++ b/fiftyone/operators/utils.py @@ -0,0 +1,50 @@ +""" +FiftyOne operator utilities. + +| Copyright 2017-2023, Voxel51, Inc. +| `voxel51.com `_ +| +""" +import logging + + +class ProgressHandler(logging.Handler): + """A logging handler that reports all logging messages issued while the + handler's context manager is active to the provided execution context's + :meth:`set_progress() ` + method. + + Args: + ctx: an :class:`fiftyone.operators.executor.ExecutionContext` + logger (None): a specific ``logging.Logger`` for which to report + records. By default, the root logger is used + level (None): an optional logging level above which to report records. + By default, the logger's effective level is used + """ + + def __init__(self, ctx, logger=None, level=None): + super().__init__() + self.ctx = ctx + self.logger = logger + self.level = level + + def __enter__(self): + if self.logger is None: + self.logger = logging.getLogger() + + if self.level is None: + self.level = self.logger.getEffectiveLevel() + + self.setLevel(self.level) + self.logger.addHandler(self) + + def __exit__(self, *args): + try: + self.logger.removeHandler(self) + except: + pass + + def emit(self, record): + msg = self.format(record) + print(f"****** {msg} ******") + self.ctx.set_progress(label=msg)