Skip to content

Commit

Permalink
Update logging (facebook#2815)
Browse files Browse the repository at this point in the history
Summary:
Pull Request resolved: facebook#2815

Make the "fetching" log less confusing: no fetching is actually taking place when `trials` is empty! Also, log the checking of completion criterion if GSS is set.

Reviewed By: Balandat

Differential Revision: D63798514

fbshipit-source-id: 0eaf6524fa43f8f1c8e6af32a9dda014ca475f89
  • Loading branch information
Lena Kashtelyan authored and facebook-github-bot committed Oct 3, 2024
1 parent 780ce7f commit 4422ada
Showing 1 changed file with 15 additions and 4 deletions.
19 changes: 15 additions & 4 deletions ax/service/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,14 @@ def completion_criterion(self) -> tuple[bool, str]:
and self.options.global_stopping_strategy is not None
):
gss = none_throws(self.options.global_stopping_strategy)
if (num_trials := len(self.trials)) > 1000:
# When there are many trials, checking the global stopping
# strategy can get a little bit slow, so we log when we start it,
# to avoid user confusion and to keep a record of the run times.
self.logger.info(
f"There are {num_trials} trials; performing "
f"completion criterion check with {gss}..."
)
stop_optimization, global_stopping_msg = gss.should_stop_optimization(
experiment=self.experiment
)
Expand Down Expand Up @@ -1006,8 +1014,8 @@ def run_trials_and_yield_results(
# schedule new trials and poll existing ones in a loop.
self._num_remaining_requested_trials = max_trials
while (
not self.should_consider_optimization_complete()[0]
and self._num_remaining_requested_trials > 0
self._num_remaining_requested_trials > 0
and not self.should_consider_optimization_complete()[0]
):
if self.should_abort_optimization():
yield self._abort_optimization(num_preexisting_trials=n_existing)
Expand Down Expand Up @@ -1471,8 +1479,11 @@ def _get_trial_indices_to_fetch(
- prev_completed_trial_idcs
)
idcs = make_indices_str(indices=newly_completed)
self.logger.info(f"Fetching data for newly completed trials: {idcs}.")
trial_indices_to_fetch.update(newly_completed)
if newly_completed:
self.logger.info(f"Fetching data for newly completed trials: {idcs}.")
trial_indices_to_fetch.update(newly_completed)
else:
self.logger.info("No newly completed trials; not fetching data for any.")

# Fetch data for running trials that have metrics available while running
if (
Expand Down

0 comments on commit 4422ada

Please sign in to comment.