debug

kubeflow · Apr 15, 2020 · b9ed143 · b9ed143
1 parent b43218e
commit b9ed143
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 9 deletions.
diff --git a/pkg/controller.v1alpha3/trial/managerclient/managerclient.go b/pkg/controller.v1alpha3/trial/managerclient/managerclient.go
@@ -35,6 +35,30 @@ func (d *DefaultClient) GetTrialObservationLog(
 	if err != nil {
 		return nil, err
 	}
+	if reply.ObservationLog == nil {
+		reply.ObservationLog = &api_pb.ObservationLog{}
+	}
+	if reply.ObservationLog.MetricLogs == nil {
+		reply.ObservationLog.MetricLogs = []*api_pb.MetricLog{}
+	}
+
+	metricLogs := reply.ObservationLog.MetricLogs
+	for _, metricName := range instance.Spec.Objective.AdditionalMetricNames {
+		request := &api_pb.GetObservationLogRequest{
+			TrialName:  instance.Name, MetricName: metricName,
+		}
+		reply, err := common.GetObservationLog(request)
+		if err != nil {
+			return nil, err
+		}
+		if reply.ObservationLog == nil || reply.ObservationLog.MetricLogs == nil {
+			continue
+		}
+		for _, log := range reply.ObservationLog.MetricLogs {
+			metricLogs = append(metricLogs, log)
+		}
+	}
+
 	return reply, nil
 }
 

diff --git a/pkg/controller.v1alpha3/trial/trial_controller_util.go b/pkg/controller.v1alpha3/trial/trial_controller_util.go
@@ -201,12 +201,12 @@ func getMetrics(metricLogs []*api_pb.MetricLog, strategies map[string]commonv1al
 		}
 	}
 
-	observation := commonv1alpha3.Observation{}
+	observation := &commonv1alpha3.Observation{}
 	for _, metric := range metrics {
 		observation.Metrics = append(observation.Metrics, *metric)
 	}
 
-	return &observation, nil
+	return observation, nil
 }
 
 func needUpdateFinalizers(trial *trialsv1alpha3.Trial) (bool, []string) {

diff --git a/test/e2e/v1alpha3/run-e2e-experiment.go b/test/e2e/v1alpha3/run-e2e-experiment.go
@@ -73,13 +73,6 @@ func main() {
 		if err != nil {
 			log.Fatal("Get Experiment error ", err)
 		}
-		log.Println(exp.Spec.Objective)
-		log.Println(exp.Spec.Algorithm)
-		for _, name := range exp.Status.RunningTrialList {
-			trial, _ := kclient.GetTrial(name, exp.Namespace)
-			log.Println(trial.Spec.Objective)
-			log.Println(trial.Status.Observation)
-		}
 		log.Printf("Waiting for Experiment %s to finish.", exp.Name)
 		log.Printf(`Experiment %s's trials: %d trials, %d pending trials,
 %d running trials, %d killed trials, %d succeeded trials, %d failed trials.`,