Add e2e test to presubmit (kubeflow#652)

* Adding grid e2e test to presubmit

* Adding extra checks

test/e2e/v1alpha2/run-e2e-experiment.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"time"
+
+	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	experimentsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2"
+	"github.com/kubeflow/katib/pkg/util/v1alpha2/katibclient"
+)
+
+const (
+	timeout = 20 * time.Minute
+)
+
+func verifyResult(exp *experimentsv1alpha2.Experiment) error {
+	if len(exp.Status.CurrentOptimalTrial.ParameterAssignments) == 0 {
+		return fmt.Errorf("Best parameter assignments not updated in status")
+	}
+
+	if len(exp.Status.CurrentOptimalTrial.Observation.Metrics) == 0 {
+		return fmt.Errorf("Bst metrics not updated in status")
+	}
+
+	metric := exp.Status.CurrentOptimalTrial.Observation.Metrics[0]
+	if metric.Name != exp.Spec.Objective.ObjectiveMetricName {
+		return fmt.Errorf("Best objective metric not updated in status")
+	}
+	return nil
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatal("Experiment name is missing")
+	}
+	expName := os.Args[1]
+	b, err := ioutil.ReadFile(expName)
+	if err != nil {
+		log.Fatal("Error in reading file ", err)
+	}
+	exp := &experimentsv1alpha2.Experiment{}
+	buf := bytes.NewBufferString(string(b))
+	if err = k8syaml.NewYAMLOrJSONDecoder(buf, 1024).Decode(exp); err != nil {
+		log.Fatal("Yaml decode error ", err)
+	}
+	kclient, err := katibclient.NewClient(client.Options{})
+	if err != nil {
+		log.Fatal("NewClient for Katib failed: ", err)
+	}
+	var maxtrials int32 = 3
+	var paralleltrials int32 = 2
+	exp.Spec.MaxTrialCount = &maxtrials
+	exp.Spec.ParallelTrialCount = &paralleltrials
+	err = kclient.CreateExperiment(exp)
+	if err != nil {
+		log.Fatal("CreateExperiment from YAML failed: ", err)
+	}
+
+	for endTime := time.Now().Add(timeout); time.Now().Before(endTime); {
+		exp, err = kclient.GetExperiment(exp.Name, exp.Namespace)
+		if err != nil {
+			log.Fatal("Get Experiment error ", err)
+		}
+		if exp.IsCompleted() {
+			log.Printf("Job %v finished", exp.Name)
+			break
+		}
+		log.Printf("Waiting for job %v to finish. [ %v trials running %v succeeded ]", exp.Name, exp.Status.TrialsRunning, exp.Status.TrialsSucceeded)
+		time.Sleep(20 * time.Second)
+	}
+
+	if !exp.IsCompleted() {
+		log.Fatal("Experiment run timed out")
+	}
+
+	if exp.Status.Trials != *exp.Spec.MaxTrialCount {
+		log.Fatal("All trials are not run in the experiment ", exp.Status.Trials, exp.Spec.MaxTrialCount)
+	}
+
+	if exp.Status.TrialsSucceeded != *exp.Spec.MaxTrialCount {
+		log.Fatal("All trials are not successful ", exp.Status.TrialsSucceeded, *exp.Spec.MaxTrialCount)
+	}
+	err = verifyResult(exp)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Printf("Experiment has recorded best current Optimal Trial %v", exp.Status.CurrentOptimalTrial)
+	err = kclient.DeleteExperiment(exp)
+	if err != nil {
+		log.Printf("CreateExperiment from YAML failed: %v", err)
+		return
+	}
+	log.Printf("Experiment %v deleted", exp.Name)
+}

test/scripts/v1alpha2/run-tests.sh

@@ -66,6 +66,7 @@ echo "REPO_NAME ${REPO_NAME}"
 echo "VERSION ${VERSION}"
 
 sed -i -e "s@image: katib\/v1alpha2\/katib-controller@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-controller:${VERSION}@" manifests/v1alpha2/katib-controller/katib-controller.yaml
+sed -i -e "s@image: katib\/v1alpha2\/metrics-collector@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/metrics-collector:${VERSION}@" manifests/v1alpha2/katib-controller/metricsControllerConfigMap.yaml
 sed -i -e "s@image: katib\/v1alpha2\/katib-manager@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager:${VERSION}@" manifests/v1alpha2/manager/deployment.yaml
 sed -i -e "s@image: katib\/v1alpha2\/katib-manager-rest@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager-rest:${VERSION}@" manifests/v1alpha2/manager-rest/deployment.yaml
 sed -i -e "s@image: katib\/v1alpha2\/katib-ui@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-ui:${VERSION}@" manifests/v1alpha2/ui/deployment.yaml
@@ -93,6 +94,7 @@ done
 
 echo "All Katib components are running."
 kubectl version
+kubectl cluster-info
 echo "Katib deployments"
 kubectl -n kubeflow get deploy
 echo "Katib services"
@@ -101,7 +103,7 @@ echo "Katib pods"
 kubectl -n kubeflow get pod
 
 mkdir -p ${GO_DIR}
-cp -r test ${GO_DIR}/test
+cp -r . ${GO_DIR}/
 cp -r pkg/api/v1alpha2/python/* ${GO_DIR}/test/e2e/v1alpha2
 cd ${GO_DIR}/test/e2e/v1alpha2
 kubectl apply -f valid-experiment.yaml
@@ -126,4 +128,7 @@ python get-pip.py
 pip install -r test_requirements.txt
 python test-katib-manager.py
 
+echo "Running e2e grid experiment"
+export KUBECONFIG=$HOME/.kube/config
+go run run-e2e-experiment.go ../../../examples/v1alpha2/grid-example.yaml
 exit 0

test/workflows/components/workflows-v1alpha2.libsonnet

@@ -261,7 +261,7 @@
                 ],
                 [
                   {
-                    name: "run-tests",
+                    name: "run-e2e-tests",
                     template: "run-tests",
                   },
                 ],