From b323afc128e3b8a61b8e40e049276d3fbfae98b3 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Wed, 15 Jan 2020 01:25:32 -0600 Subject: [PATCH] fix: First check failed condition (#1015) Signed-off-by: Ce Gao --- .../experiment/util/status_util.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/controller.v1alpha3/experiment/util/status_util.go b/pkg/controller.v1alpha3/experiment/util/status_util.go index 09b08736dbf..26118cd556a 100644 --- a/pkg/controller.v1alpha3/experiment/util/status_util.go +++ b/pkg/controller.v1alpha3/experiment/util/status_util.go @@ -139,6 +139,7 @@ func getObjectiveMetricValue(trial trialsv1alpha3.Trial, objectiveMetricName str return nil } +// UpdateExperimentStatusCondition updates the experiment status. func UpdateExperimentStatusCondition(collector *ExperimentsCollector, instance *experimentsv1alpha3.Experiment, isObjectiveGoalReached bool, getSuggestionDone bool) { completedTrialsCount := instance.Status.TrialsSucceeded + instance.Status.TrialsFailed + instance.Status.TrialsKilled @@ -153,6 +154,16 @@ func UpdateExperimentStatusCondition(collector *ExperimentsCollector, instance * return } + // First check if MaxFailedTrialCount is reached. + if (instance.Spec.MaxFailedTrialCount != nil) && (failedTrialsCount > *instance.Spec.MaxFailedTrialCount) { + msg := "Experiment has failed because max failed count has reached" + instance.MarkExperimentStatusFailed(ExperimentFailedReason, msg) + instance.Status.CompletionTime = &now + collector.IncreaseExperimentsFailedCount(instance.Namespace) + return + } + + // Then Check if MaxTrialCount is reached. if (instance.Spec.MaxTrialCount != nil) && (completedTrialsCount >= *instance.Spec.MaxTrialCount) { msg := "Experiment has succeeded because max trial count has reached" instance.MarkExperimentStatusSucceeded(ExperimentMaxTrialsReachedReason, msg) @@ -169,14 +180,6 @@ func UpdateExperimentStatusCondition(collector *ExperimentsCollector, instance * return } - if (instance.Spec.MaxFailedTrialCount != nil) && (failedTrialsCount > *instance.Spec.MaxFailedTrialCount) { - msg := "Experiment has failed because max failed count has reached" - instance.MarkExperimentStatusFailed(ExperimentFailedReason, msg) - instance.Status.CompletionTime = &now - collector.IncreaseExperimentsFailedCount(instance.Namespace) - return - } - msg := "Experiment is running" instance.MarkExperimentStatusRunning(ExperimentRunningReason, msg) }