From 4422adadbeb9068659dac23569a91a11d10b4ebc Mon Sep 17 00:00:00 2001 From: Lena Kashtelyan Date: Thu, 3 Oct 2024 10:07:15 -0700 Subject: [PATCH] Update logging (#2815) Summary: Pull Request resolved: https://github.com/facebook/Ax/pull/2815 Make the "fetching" log less confusing: no fetching is actually taking place when `trials` is empty! Also, log the checking of completion criterion if GSS is set. Reviewed By: Balandat Differential Revision: D63798514 fbshipit-source-id: 0eaf6524fa43f8f1c8e6af32a9dda014ca475f89 --- ax/service/scheduler.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ax/service/scheduler.py b/ax/service/scheduler.py index 3ec81ebd82b..46360925975 100644 --- a/ax/service/scheduler.py +++ b/ax/service/scheduler.py @@ -499,6 +499,14 @@ def completion_criterion(self) -> tuple[bool, str]: and self.options.global_stopping_strategy is not None ): gss = none_throws(self.options.global_stopping_strategy) + if (num_trials := len(self.trials)) > 1000: + # When there are many trials, checking the global stopping + # strategy can get a little bit slow, so we log when we start it, + # to avoid user confusion and to keep a record of the run times. + self.logger.info( + f"There are {num_trials} trials; performing " + f"completion criterion check with {gss}..." + ) stop_optimization, global_stopping_msg = gss.should_stop_optimization( experiment=self.experiment ) @@ -1006,8 +1014,8 @@ def run_trials_and_yield_results( # schedule new trials and poll existing ones in a loop. self._num_remaining_requested_trials = max_trials while ( - not self.should_consider_optimization_complete()[0] - and self._num_remaining_requested_trials > 0 + self._num_remaining_requested_trials > 0 + and not self.should_consider_optimization_complete()[0] ): if self.should_abort_optimization(): yield self._abort_optimization(num_preexisting_trials=n_existing) @@ -1471,8 +1479,11 @@ def _get_trial_indices_to_fetch( - prev_completed_trial_idcs ) idcs = make_indices_str(indices=newly_completed) - self.logger.info(f"Fetching data for newly completed trials: {idcs}.") - trial_indices_to_fetch.update(newly_completed) + if newly_completed: + self.logger.info(f"Fetching data for newly completed trials: {idcs}.") + trial_indices_to_fetch.update(newly_completed) + else: + self.logger.info("No newly completed trials; not fetching data for any.") # Fetch data for running trials that have metrics available while running if (