From f4950bc76d2da09a9c97b8a2a343c01bf4a2f3bc Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 1 Apr 2020 22:17:42 -0700
Subject: [PATCH 01/20] break down kverify package to files

---
 .../bootstrapper/bsutil/kverify/api_server.go | 173 ++++++++++++++++++
 .../bootstrapper/bsutil/kverify/default_sa.go |  56 ++++++
 .../bootstrapper/bsutil/kverify/kverify.go    | 173 +++++++++---------
 .../bsutil/kverify/system_pods.go             | 114 ++++++++++++
 4 files changed, 428 insertions(+), 88 deletions(-)
 create mode 100644 pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
 create mode 100644 pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
 create mode 100644 pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
new file mode 100644
index 000000000000..3a4e3a871b3f
--- /dev/null
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
@@ -0,0 +1,173 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kverify verifies a running kubernetes cluster is healthy
+package kverify
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"os/exec"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/util/wait"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+)
+
+// WaitForHealthyAPIServer waits for api server status to be running
+func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, start time.Time, ip string, port int, timeout time.Duration) error {
+	glog.Infof("waiting for apiserver healthz status ...")
+	hStart := time.Now()
+
+	healthz := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during healthz check")
+		}
+
+		if time.Since(start) > minLogCheckTime {
+			announceProblems(r, bs, cfg, cr)
+			time.Sleep(kconst.APICallRetryInterval * 5)
+		}
+
+		status, err := apiServerHealthz(net.ParseIP(ip), port)
+		if err != nil {
+			glog.Warningf("status: %v", err)
+			return false, nil
+		}
+		if status != state.Running {
+			return false, nil
+		}
+		return true, nil
+	}
+
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil {
+		return fmt.Errorf("apiserver healthz never reported healthy")
+	}
+	glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart))
+	return nil
+}
+
+// WaitForAPIServerProcess waits for api server to be healthy returns error if it doesn't
+func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, start time.Time, timeout time.Duration) error {
+	glog.Infof("waiting for apiserver process to appear ...")
+	err := wait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during process check")
+		}
+
+		if time.Since(start) > minLogCheckTime {
+			announceProblems(r, bs, cfg, cr)
+			time.Sleep(kconst.APICallRetryInterval * 5)
+		}
+
+		if _, ierr := apiServerPID(cr); ierr != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("apiserver process never appeared")
+	}
+	glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start))
+	return nil
+}
+
+// apiServerPID returns our best guess to the apiserver pid
+func apiServerPID(cr command.Runner) (int, error) {
+	rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "-xnf", "kube-apiserver.*minikube.*"))
+	if err != nil {
+		return 0, err
+	}
+	s := strings.TrimSpace(rr.Stdout.String())
+	return strconv.Atoi(s)
+}
+
+// APIServerStatus returns apiserver status in libmachine style state.State
+func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error) {
+	glog.Infof("Checking apiserver status ...")
+
+	pid, err := apiServerPID(cr)
+	if err != nil {
+		glog.Warningf("stopped: unable to get apiserver pid: %v", err)
+		return state.Stopped, nil
+	}
+
+	// Get the freezer cgroup entry for this pid
+	rr, err := cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", fmt.Sprintf("/proc/%d/cgroup", pid)))
+	if err != nil {
+		glog.Warningf("unable to find freezer cgroup: %v", err)
+		return apiServerHealthz(ip, port)
+
+	}
+	freezer := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("apiserver freezer: %q", freezer)
+	fparts := strings.Split(freezer, ":")
+	if len(fparts) != 3 {
+		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
+		return apiServerHealthz(ip, port)
+	}
+
+	rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
+	if err != nil {
+		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
+		return apiServerHealthz(ip, port)
+	}
+
+	fs := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("freezer state: %q", fs)
+	if fs == "FREEZING" || fs == "FROZEN" {
+		return state.Paused, nil
+	}
+	return apiServerHealthz(ip, port)
+}
+
+// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State
+func apiServerHealthz(ip net.IP, port int) (state.State, error) {
+	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(port)))
+	glog.Infof("Checking apiserver healthz at %s ...", url)
+	// To avoid: x509: certificate signed by unknown authority
+	tr := &http.Transport{
+		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(url)
+	// Connection refused, usually.
+	if err != nil {
+		glog.Infof("stopped: %s: %v", url, err)
+		return state.Stopped, nil
+	}
+	if resp.StatusCode == http.StatusUnauthorized {
+		glog.Errorf("%s returned code %d (unauthorized). Please ensure that your apiserver authorization settings make sense!", url, resp.StatusCode)
+		return state.Error, nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		glog.Warningf("%s response: %v %+v", url, err, resp)
+		return state.Error, nil
+	}
+	return state.Running, nil
+}
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
new file mode 100644
index 000000000000..214b097f732e
--- /dev/null
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
@@ -0,0 +1,56 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kverify verifies a running kubernetes cluster is healthy
+package kverify
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+// WaitForDefaultSA waits for the default service account to be created.
+func WaitForDefaultSA(cs *kubernetes.Clientset) error {
+	glog.Info("waiting for default service account to be created ...")
+	pStart := time.Now()
+	saReady := func() error {
+		// equivalent to manual check of 'kubectl --context profile get serviceaccount default'
+		sas, err := cs.CoreV1().ServiceAccounts("default").List(meta.ListOptions{})
+		if err != nil {
+			glog.Infof("temproary error waiting for default SA: %v", err)
+			return err
+		}
+		for _, sa := range sas.Items {
+			if sa.Name == "default" {
+				glog.Infof("found service account: %q", sa.Name)
+				return nil
+			}
+		}
+		return fmt.Errorf("couldn't find default service account")
+	}
+	if err := retry.Expo(saReady, 500*time.Millisecond, 30*time.Second); err != nil {
+		return errors.Wrapf(err, "waited %s for SA", time.Since(pStart))
+	}
+
+	glog.Infof("duration metric: took %s for default service account to be created ...", time.Since(pStart))
+	return nil
+}
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index b7bf7fd5c5a1..40a4e96bfc85 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors All rights reserved.
+Copyright 2019 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,24 +18,14 @@ limitations under the License.
 package kverify
 
 import (
-	"crypto/tls"
 	"fmt"
-	"net"
-	"net/http"
 	"os/exec"
-	"path"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
-	"github.com/pkg/errors"
 	core "k8s.io/api/core/v1"
-	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apimachinery/pkg/version"
-	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -44,86 +34,23 @@ import (
 	"k8s.io/minikube/pkg/minikube/logs"
 )
 
-// minLogCheckTime how long to wait before spamming error logs to console
-const minLogCheckTime = 60 * time.Second
-
-// WaitForAPIServerProcess waits for api server to be healthy returns error if it doesn't
-func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, start time.Time, timeout time.Duration) error {
-	glog.Infof("waiting for apiserver process to appear ...")
-	err := wait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) {
-		if time.Since(start) > timeout {
-			return false, fmt.Errorf("cluster wait timed out during process check")
-		}
-
-		if time.Since(start) > minLogCheckTime {
-			announceProblems(r, bs, cfg, cr)
-			time.Sleep(kconst.APICallRetryInterval * 5)
-		}
-
-		if _, ierr := apiServerPID(cr); ierr != nil {
-			return false, nil
-		}
-
-		return true, nil
-	})
-	if err != nil {
-		return fmt.Errorf("apiserver process never appeared")
-	}
-	glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start))
-	return nil
-}
-
-// apiServerPID returns our best guess to the apiserver pid
-func apiServerPID(cr command.Runner) (int, error) {
-	rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "-xnf", "kube-apiserver.*minikube.*"))
-	if err != nil {
-		return 0, err
-	}
-	s := strings.TrimSpace(rr.Stdout.String())
-	return strconv.Atoi(s)
-}
-
-// ExpectedComponentsRunning returns whether or not all expected components are running
-func ExpectedComponentsRunning(cs *kubernetes.Clientset) error {
-	expected := []string{
-		"kube-dns", // coredns
-		"etcd",
-		"kube-apiserver",
-		"kube-controller-manager",
-		"kube-proxy",
-		"kube-scheduler",
-	}
-
-	found := map[string]bool{}
+const (
+	// APIServer is the name used in the flags for k8s api server
+	APIServer = "apiserver"
+	// SystemPod is the name used in the flags for pods in the kube system
+	SystemPods = "system_pods"
+	// DefaultSA is the name used in the flags for default service account
+	DefaultSA = "default_sa"
+)
 
-	pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{})
-	if err != nil {
-		return err
-	}
+// DefaultWaits is the default components to wait for
+var DefaultWaits = []string{APIServer, SystemPods}
 
-	for _, pod := range pods.Items {
-		glog.Infof("found pod: %s", podStatusMsg(pod))
-		if pod.Status.Phase != core.PodRunning {
-			continue
-		}
-		for k, v := range pod.ObjectMeta.Labels {
-			if k == "component" || k == "k8s-app" {
-				found[v] = true
-			}
-		}
-	}
+// AvailableWaits is list of possible components that user can choose to wait for
+var AvailableWaits = []string{APIServer, SystemPods, DefaultSA}
 
-	missing := []string{}
-	for _, e := range expected {
-		if !found[e] {
-			missing = append(missing, e)
-		}
-	}
-	if len(missing) > 0 {
-		return fmt.Errorf("missing components: %v", strings.Join(missing, ", "))
-	}
-	return nil
-}
+// minLogCheckTime how long to wait before spamming error logs to console
+const minLogCheckTime = 30 * time.Second
 
 // podStatusMsg returns a human-readable pod status, for generating debug status
 func podStatusMsg(pod core.Pod) string {
@@ -253,6 +180,7 @@ func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg conf
 	}
 }
 
+<<<<<<< HEAD
 // APIServerStatus returns apiserver status in libmachine style state.State
 func APIServerStatus(cr command.Runner, hostname string, port int) (state.State, error) {
 	glog.Infof("Checking apiserver status ...")
@@ -319,6 +247,75 @@ func apiServerHealthz(hostname string, port int) (state.State, error) {
 	return state.Running, nil
 }
 
+||||||| merged common ancestors
+// APIServerStatus returns apiserver status in libmachine style state.State
+func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error) {
+	glog.Infof("Checking apiserver status ...")
+
+	pid, err := apiServerPID(cr)
+	if err != nil {
+		glog.Warningf("stopped: unable to get apiserver pid: %v", err)
+		return state.Stopped, nil
+	}
+
+	// Get the freezer cgroup entry for this pid
+	rr, err := cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", fmt.Sprintf("/proc/%d/cgroup", pid)))
+	if err != nil {
+		glog.Warningf("unable to find freezer cgroup: %v", err)
+		return apiServerHealthz(ip, port)
+
+	}
+	freezer := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("apiserver freezer: %q", freezer)
+	fparts := strings.Split(freezer, ":")
+	if len(fparts) != 3 {
+		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
+		return apiServerHealthz(ip, port)
+	}
+
+	rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
+	if err != nil {
+		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
+		return apiServerHealthz(ip, port)
+	}
+
+	fs := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("freezer state: %q", fs)
+	if fs == "FREEZING" || fs == "FROZEN" {
+		return state.Paused, nil
+	}
+	return apiServerHealthz(ip, port)
+}
+
+// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State
+func apiServerHealthz(ip net.IP, port int) (state.State, error) {
+	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(port)))
+	glog.Infof("Checking apiserver healthz at %s ...", url)
+	// To avoid: x509: certificate signed by unknown authority
+	tr := &http.Transport{
+		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(url)
+	// Connection refused, usually.
+	if err != nil {
+		glog.Infof("stopped: %s: %v", url, err)
+		return state.Stopped, nil
+	}
+	if resp.StatusCode == http.StatusUnauthorized {
+		glog.Errorf("%s returned code %d (unauthorized). Please ensure that your apiserver authorization settings make sense!", url, resp.StatusCode)
+		return state.Error, nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		glog.Warningf("%s response: %v %+v", url, err, resp)
+		return state.Error, nil
+	}
+	return state.Running, nil
+}
+
+=======
+>>>>>>> break down kverify package to files
 // KubeletStatus checks the kubelet status
 func KubeletStatus(cr command.Runner) (state.State, error) {
 	glog.Infof("Checking kubelet status ...")
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
new file mode 100644
index 000000000000..700b8f2d8370
--- /dev/null
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
@@ -0,0 +1,114 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kverify verifies a running kubernetes cluster is healthy
+package kverify
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+)
+
+// WaitForSystemPods verifies essential pods for running kurnetes is running
+func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
+	glog.Info("waiting for kube-system pods to appear ...")
+	pStart := time.Now()
+
+	podList := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during pod check")
+		}
+		if time.Since(start) > minLogCheckTime {
+			announceProblems(r, bs, cfg, cr)
+			time.Sleep(kconst.APICallRetryInterval * 5)
+		}
+
+		// Wait for any system pod, as waiting for apiserver may block until etcd
+		pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+		if err != nil {
+			glog.Warningf("pod list returned error: %v", err)
+			return false, nil
+		}
+		glog.Infof("%d kube-system pods found", len(pods.Items))
+		for _, pod := range pods.Items {
+			glog.Infof(podStatusMsg(pod))
+		}
+
+		if len(pods.Items) < 2 {
+			return false, nil
+		}
+		return true, nil
+	}
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil {
+		return fmt.Errorf("apiserver never returned a pod list")
+	}
+	glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
+	return nil
+}
+
+// ExpectedComponentsRunning returns whether or not all expected components are running
+func ExpectedComponentsRunning(cs *kubernetes.Clientset) error {
+	expected := []string{
+		"kube-dns", // coredns
+		"etcd",
+		"kube-apiserver",
+		"kube-controller-manager",
+		"kube-proxy",
+		"kube-scheduler",
+	}
+
+	found := map[string]bool{}
+
+	pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, pod := range pods.Items {
+		glog.Infof("found pod: %s", podStatusMsg(pod))
+		if pod.Status.Phase != core.PodRunning {
+			continue
+		}
+		for k, v := range pod.ObjectMeta.Labels {
+			if k == "component" || k == "k8s-app" {
+				found[v] = true
+			}
+		}
+	}
+
+	missing := []string{}
+	for _, e := range expected {
+		if !found[e] {
+			missing = append(missing, e)
+		}
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("missing components: %v", strings.Join(missing, ", "))
+	}
+	return nil
+}

From e4eae47e8695f9992f13d885e7ff8413d0fd0e98 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 1 Apr 2020 23:28:40 -0700
Subject: [PATCH 02/20] interpret wait flags

---
 cmd/minikube/cmd/start.go                     | 47 ++++++++++++++++++-
 .../bootstrapper/bsutil/kverify/kverify.go    | 29 ++++++++----
 2 files changed, 64 insertions(+), 12 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 16a3140279f3..eac228b10be0 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -44,6 +44,7 @@ import (
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -109,7 +110,7 @@ const (
 	downloadOnly            = "download-only"
 	dnsProxy                = "dns-proxy"
 	hostDNSResolver         = "host-dns-resolver"
-	waitUntilHealthy        = "wait"
+	waitComponents          = "wait"
 	force                   = "force"
 	dryRun                  = "dry-run"
 	interactive             = "interactive"
@@ -171,7 +172,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
-	startCmd.Flags().Bool(waitUntilHealthy, true, "Block until the apiserver is servicing API requests")
+	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitsKeys, fmt.Sprintf("comma separated list of kuberentes components to wait to verify after start. defaults to %q, available options: %q . can also specify 'all' or 'none'", kverify.DefaultWaitsKeys, kverify.AllWaitsKeys))
 	startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
 	startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
@@ -1200,3 +1201,45 @@ func getKubernetesVersion(old *config.ClusterConfig) string {
 	}
 	return nv
 }
+
+// interpretWaitFlag interprets the wait flag and respects the legacy minikube users
+// returns 	waitForAPI, waitForSysPod, waitForSA
+func interpretWaitFlag(cmd cobra.Command) map[string]bool {
+	if !cmd.Flags().Changed(waitComponents) {
+		glog.Infof("Wait Components : %+v", kverify.DefaultWaits)
+		return kverify.DefaultWaits
+	}
+
+	waitFlags, err := cmd.Flags().GetStringSlice(waitComponents)
+	if err != nil {
+		glog.Infof("failed to get wait from flags, will use default wait components : %+v", kverify.DefaultWaits)
+		return kverify.DefaultWaits
+	}
+
+	// before minikube 1.9.0, wait flag was boolean
+	if (len(waitFlags) == 1 && waitFlags[0] == "true") || (len(waitFlags) == 1 && waitFlags[0] == "all") {
+		return kverify.AllWaitsCompo
+	}
+
+	// respecting legacy flag format --wait=false
+	// before minikube 1.9.0, wait flag was boolean
+	if (len(waitFlags) == 1 && waitFlags[0] == "false") || len(waitFlags) == 1 && waitFlags[0] == "none" {
+		return kverify.NoWaitsCompo
+	}
+
+	waitCompos := map[string]bool{}
+	for _, wc := range waitFlags {
+		seen := false
+		for _, valid := range kverify.AllValidWaitsList {
+			if wc == valid {
+				waitCompos[wc] = true
+				seen = true
+				continue
+			}
+		}
+		if !seen {
+			glog.Warning("invalid wait component flag %q. valid options are %q", wc, strings.Join(kverify.AllValidWaitsList, ","))
+		}
+	}
+	return waitCompos
+}
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 40a4e96bfc85..806a07b19946 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -35,19 +35,28 @@ import (
 )
 
 const (
-	// APIServer is the name used in the flags for k8s api server
-	APIServer = "apiserver"
-	// SystemPod is the name used in the flags for pods in the kube system
-	SystemPods = "system_pods"
-	// DefaultSA is the name used in the flags for default service account
-	DefaultSA = "default_sa"
+	// APIServerWait is the name used in the flags for k8s api server
+	APIServerWait = "apiserver"
+	// SystemPodsWait is the name used in the flags for pods in the kube system
+	SystemPodsWait = "system_pods"
+	// DefaultSAWait is the name used in the flags for default service account
+	DefaultSAWait = "default_sa"
 )
 
-// DefaultWaits is the default components to wait for
-var DefaultWaits = []string{APIServer, SystemPods}
+// DefaultWaits is map of the the default components to wait for
+var DefaultWaits = map[string]bool{APIServerWait: true, SystemPodsWait: true}
 
-// AvailableWaits is list of possible components that user can choose to wait for
-var AvailableWaits = []string{APIServer, SystemPods, DefaultSA}
+// DefaultWaitsKeys is list of all default components to wait for
+var DefaultWaitsKeys = []string{APIServerWait, SystemPodsWait}
+
+// NoWaitsCompo is map of componets to wait for if specified 'none' or 'false'
+var NoWaitsCompo = map[string]bool{}
+
+// AllWaitsCompo is map for waiting for all components.
+var AllWaitsCompo = map[string]bool{APIServerWait: true, SystemPodsWait: true, DefaultSAWait: true}
+
+// AllValidWaitsList list of all valid components to wait for
+var AllValidWaitsList = []string{APIServerWait, SystemPodsWait, DefaultSAWait}
 
 // minLogCheckTime how long to wait before spamming error logs to console
 const minLogCheckTime = 30 * time.Second

From 3b39b271aca93c6176b55569617f97c6f1430ede Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 1 Apr 2020 23:30:27 -0700
Subject: [PATCH 03/20] fix interpret wait

---
 cmd/minikube/cmd/start.go                           | 6 +++---
 pkg/minikube/bootstrapper/bsutil/kverify/kverify.go | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index eac228b10be0..93bc8c4e3385 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1203,7 +1203,7 @@ func getKubernetesVersion(old *config.ClusterConfig) string {
 }
 
 // interpretWaitFlag interprets the wait flag and respects the legacy minikube users
-// returns 	waitForAPI, waitForSysPod, waitForSA
+// returns map of components to wait for
 func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 	if !cmd.Flags().Changed(waitComponents) {
 		glog.Infof("Wait Components : %+v", kverify.DefaultWaits)
@@ -1224,10 +1224,10 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 	// respecting legacy flag format --wait=false
 	// before minikube 1.9.0, wait flag was boolean
 	if (len(waitFlags) == 1 && waitFlags[0] == "false") || len(waitFlags) == 1 && waitFlags[0] == "none" {
-		return kverify.NoWaitsCompo
+		return kverify.NoWaitsCompos
 	}
 
-	waitCompos := map[string]bool{}
+	waitCompos := kverify.NoWaitsCompos
 	for _, wc := range waitFlags {
 		seen := false
 		for _, valid := range kverify.AllValidWaitsList {
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 806a07b19946..4987e82b5822 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -49,8 +49,8 @@ var DefaultWaits = map[string]bool{APIServerWait: true, SystemPodsWait: true}
 // DefaultWaitsKeys is list of all default components to wait for
 var DefaultWaitsKeys = []string{APIServerWait, SystemPodsWait}
 
-// NoWaitsCompo is map of componets to wait for if specified 'none' or 'false'
-var NoWaitsCompo = map[string]bool{}
+// NoWaitsCompos is map of componets to wait for if specified 'none' or 'false'
+var NoWaitsCompos = map[string]bool{}
 
 // AllWaitsCompo is map for waiting for all components.
 var AllWaitsCompo = map[string]bool{APIServerWait: true, SystemPodsWait: true, DefaultSAWait: true}

From 1df01b587c336832f90c614f46afa14f6a4cb97b Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 1 Apr 2020 23:44:17 -0700
Subject: [PATCH 04/20] add wait

---
 pkg/minikube/config/types.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 97c2a1303991..ef3b2d41cc01 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -65,6 +65,7 @@ type ClusterConfig struct {
 	KubernetesConfig        KubernetesConfig
 	Nodes                   []Node
 	Addons                  map[string]bool
+	WaitForCompos           map[string]bool // map of components to verify and wait for after start.
 }
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.

From f49a7a2c5e81812028b3b8c68e76295ed4dda938 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 00:22:39 -0700
Subject: [PATCH 05/20] add new wait logic to kubeadm

---
 .../bootstrapper/bsutil/kverify/default_sa.go |  2 +-
 .../bootstrapper/bsutil/kverify/kverify.go    | 10 ++---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  | 45 +++++++++++++------
 pkg/minikube/node/start.go                    |  2 +-
 4 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
index 214b097f732e..239d01604aa0 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Kubernetes Authors All rights reserved.
+Copyright 2016 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 4987e82b5822..6198e202086a 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -39,8 +39,8 @@ const (
 	APIServerWait = "apiserver"
 	// SystemPodsWait is the name used in the flags for pods in the kube system
 	SystemPodsWait = "system_pods"
-	// DefaultSAWait is the name used in the flags for default service account
-	DefaultSAWait = "default_sa"
+	// DefaultServiceAccountWait is the name used in the flags for default service account
+	DefaultServiceAccountWait = "default_sa"
 )
 
 // DefaultWaits is map of the the default components to wait for
@@ -50,13 +50,13 @@ var DefaultWaits = map[string]bool{APIServerWait: true, SystemPodsWait: true}
 var DefaultWaitsKeys = []string{APIServerWait, SystemPodsWait}
 
 // NoWaitsCompos is map of componets to wait for if specified 'none' or 'false'
-var NoWaitsCompos = map[string]bool{}
+var NoWaitsCompos = map[string]bool{APIServerWait: false, SystemPodsWait: false, DefaultServiceAccountWait: false}
 
 // AllWaitsCompo is map for waiting for all components.
-var AllWaitsCompo = map[string]bool{APIServerWait: true, SystemPodsWait: true, DefaultSAWait: true}
+var AllWaitsCompo = map[string]bool{APIServerWait: true, SystemPodsWait: true, DefaultServiceAccountWait: true}
 
 // AllValidWaitsList list of all valid components to wait for
-var AllValidWaitsList = []string{APIServerWait, SystemPodsWait, DefaultSAWait}
+var AllValidWaitsList = []string{APIServerWait, SystemPodsWait, DefaultServiceAccountWait}
 
 // minLogCheckTime how long to wait before spamming error logs to console
 const minLogCheckTime = 30 * time.Second
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 63fcab2ebde5..49881c61cc03 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -341,38 +341,55 @@ func (k *Bootstrapper) client(ip string, port int) (*kubernetes.Clientset, error
 // WaitForNode blocks until the node appears to be healthy
 func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, timeout time.Duration) error {
 	start := time.Now()
-
 	if !n.ControlPlane {
 		glog.Infof("%s is not a control plane, nothing to wait for", n.Name)
 		return nil
 	}
+	if cfg.WaitForCompos == kverify.NoWaitsCompos {
+		glog.Infof("skip waiting for components based on config.")
+		return nil
+	}
 
 	cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c})
 	if err != nil {
-		return err
-	}
-
-	if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil {
-		return err
+		return errors.Wrap(err, "new cruntime")
 	}
 
 	hostname, _, port, err := driver.ControlPaneEndpoint(&cfg, &n, cfg.Driver)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "get control plane endpoint")
 	}
 
-	client, err := k.client(hostname, port)
-	if err != nil {
-		return errors.Wrap(err, "get k8s client")
+	if cfg.WaitForCompos[kverify.APIServerWait] {
+		if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil {
+			return errors.Wrap(err, "wait for apiserver proc")
+		}
+
+		if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, start, ip, port, timeout); err != nil {
+			return errors.Wrap(err, "wait for healthy API server")
+		}
 	}
 
-	if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, start, hostname, port, timeout); err != nil {
-		return err
+	if cfg.WaitForCompos[kverify.SystemPodsWait] {
+		client, err := k.client(ip, port)
+		if err != nil {
+			return errors.Wrap(err, "get k8s client")
+		}
+		if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, client, start, timeout); err != nil {
+			return errors.Wrap(err, "waiting for system pods")
+		}
 	}
 
-	if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, client, start, timeout); err != nil {
-		return errors.Wrap(err, "waiting for system pods")
+	if cfg.WaitForCompos[kverify.DefaultServiceAccountWait] {
+		client, err := k.client(ip, port)
+		if err != nil {
+			return errors.Wrap(err, "get k8s client")
+		}
+		if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, client, start, timeout); err != nil {
+			return errors.Wrap(err, "waiting for system pods")
+		}
 	}
+	glog.Infof("duration metric: took %s to wait for : %+v ...", time.Since(start), cfg.WaitForCompos)
 	return nil
 }
 
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 11695485d00e..137cd669eecf 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -156,7 +156,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo
 		}
 
 		// Skip pre-existing, because we already waited for health
-		if viper.GetBool(waitUntilHealthy) && !preExists {
+		if !preExists {
 			if err := bs.WaitForNode(cc, n, viper.GetDuration(waitTimeout)); err != nil {
 				return nil, errors.Wrap(err, "Wait failed")
 			}

From 4c8f79d823281208be0e8666648352914e8edf71 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 00:48:40 -0700
Subject: [PATCH 06/20] add wait logic to kubeadm

---
 cmd/minikube/cmd/start.go                     |   4 +-
 .../bootstrapper/bsutil/kverify/api_server.go |  37 ++-
 .../bootstrapper/bsutil/kverify/kverify.go    | 245 +-----------------
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  |   8 +-
 4 files changed, 50 insertions(+), 244 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 93bc8c4e3385..82112c1f2105 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -172,7 +172,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
-	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitsKeys, fmt.Sprintf("comma separated list of kuberentes components to wait to verify after start. defaults to %q, available options: %q . can also specify 'all' or 'none'", kverify.DefaultWaitsKeys, kverify.AllWaitsKeys))
+	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitsKeys, fmt.Sprintf("comma separated list of kuberentes components to wait to verify after start. defaults to %q, available options: %q . can also specify 'all' or 'none'", kverify.DefaultWaitsKeys, kverify.AllValidWaitsList))
 	startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
 	startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
@@ -1218,7 +1218,7 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 
 	// before minikube 1.9.0, wait flag was boolean
 	if (len(waitFlags) == 1 && waitFlags[0] == "true") || (len(waitFlags) == 1 && waitFlags[0] == "all") {
-		return kverify.AllWaitsCompo
+		return kverify.AllWaitsCompos
 	}
 
 	// respecting legacy flag format --wait=false
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
index 3a4e3a871b3f..ae1dcc9c46df 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
@@ -30,7 +30,10 @@ import (
 
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -39,7 +42,7 @@ import (
 )
 
 // WaitForHealthyAPIServer waits for api server status to be running
-func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, start time.Time, ip string, port int, timeout time.Duration) error {
+func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, ip string, port int, timeout time.Duration) error {
 	glog.Infof("waiting for apiserver healthz status ...")
 	hStart := time.Now()
 
@@ -67,7 +70,23 @@ func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, c
 	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil {
 		return fmt.Errorf("apiserver healthz never reported healthy")
 	}
-	glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart))
+
+	vcheck := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during version check")
+		}
+		if err := APIServerVersionMatch(client, cfg.KubernetesConfig.KubernetesVersion); err != nil {
+			glog.Warningf("api server version match failed: %v", err)
+			return false, nil
+		}
+		return true, nil
+	}
+
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, vcheck); err != nil {
+		return fmt.Errorf("controlPlane never updated to %s", cfg.KubernetesConfig.KubernetesVersion)
+	}
+
+	glog.Infof("duration metric: took %s to wait for apiserver health ...", time.Since(hStart))
 	return nil
 }
 
@@ -87,6 +106,7 @@ func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, c
 		if _, ierr := apiServerPID(cr); ierr != nil {
 			return false, nil
 		}
+
 		return true, nil
 	})
 	if err != nil {
@@ -171,3 +191,16 @@ func apiServerHealthz(ip net.IP, port int) (state.State, error) {
 	}
 	return state.Running, nil
 }
+
+// APIServerVersionMatch checks if the server version matches the expected
+func APIServerVersionMatch(client *kubernetes.Clientset, expected string) error {
+	vi, err := client.ServerVersion()
+	if err != nil {
+		return errors.Wrap(err, "server version")
+	}
+	glog.Infof("control plane version: %s", vi)
+	if version.CompareKubeAwareVersionStrings(vi.String(), expected) != 0 {
+		return fmt.Errorf("controlPane = %q, expected: %q", vi.String(), expected)
+	}
+	return nil
+}
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 6198e202086a..3b6cba29882c 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -35,6 +35,8 @@ import (
 )
 
 const (
+	// minLogCheckTime how long to wait before spamming error logs to console
+	minLogCheckTime = 30 * time.Second
 	// APIServerWait is the name used in the flags for k8s api server
 	APIServerWait = "apiserver"
 	// SystemPodsWait is the name used in the flags for pods in the kube system
@@ -52,15 +54,12 @@ var DefaultWaitsKeys = []string{APIServerWait, SystemPodsWait}
 // NoWaitsCompos is map of componets to wait for if specified 'none' or 'false'
 var NoWaitsCompos = map[string]bool{APIServerWait: false, SystemPodsWait: false, DefaultServiceAccountWait: false}
 
-// AllWaitsCompo is map for waiting for all components.
-var AllWaitsCompo = map[string]bool{APIServerWait: true, SystemPodsWait: true, DefaultServiceAccountWait: true}
+// AllWaitsCompos is map for waiting for all components.
+var AllWaitsCompos = map[string]bool{APIServerWait: true, SystemPodsWait: true, DefaultServiceAccountWait: true}
 
 // AllValidWaitsList list of all valid components to wait for
 var AllValidWaitsList = []string{APIServerWait, SystemPodsWait, DefaultServiceAccountWait}
 
-// minLogCheckTime how long to wait before spamming error logs to console
-const minLogCheckTime = 30 * time.Second
-
 // podStatusMsg returns a human-readable pod status, for generating debug status
 func podStatusMsg(pod core.Pod) string {
 	var sb strings.Builder
@@ -81,103 +80,9 @@ func podStatusMsg(pod core.Pod) string {
 	return sb.String()
 }
 
-// WaitForSystemPods verifies essential pods for running kurnetes is running
-func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
-	glog.Info("waiting for kube-system pods to appear ...")
-	pStart := time.Now()
-
-	podList := func() (bool, error) {
-		if time.Since(start) > timeout {
-			return false, fmt.Errorf("cluster wait timed out during pod check")
-		}
-		if time.Since(start) > minLogCheckTime {
-			announceProblems(r, bs, cfg, cr)
-			time.Sleep(kconst.APICallRetryInterval * 5)
-		}
-
-		// Wait for any system pod, as waiting for apiserver may block until etcd
-		pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
-		if err != nil {
-			glog.Warningf("pod list returned error: %v", err)
-			return false, nil
-		}
-		glog.Infof("%d kube-system pods found", len(pods.Items))
-		for _, pod := range pods.Items {
-			glog.Infof(podStatusMsg(pod))
-		}
-
-		if len(pods.Items) < 2 {
-			return false, nil
-		}
-		return true, nil
-	}
-	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil {
-		return fmt.Errorf("apiserver never returned a pod list")
-	}
-	glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
-	return nil
-}
-
-// WaitForHealthyAPIServer waits for api server status to be running
-func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, hostname string, port int, timeout time.Duration) error {
-	glog.Infof("waiting for apiserver healthz status ...")
-	hStart := time.Now()
-
-	healthz := func() (bool, error) {
-		if time.Since(start) > timeout {
-			return false, fmt.Errorf("cluster wait timed out during healthz check")
-		}
-
-		if time.Since(start) > minLogCheckTime {
-			announceProblems(r, bs, cfg, cr)
-			time.Sleep(kconst.APICallRetryInterval * 5)
-		}
-
-		status, err := apiServerHealthz(hostname, port)
-		if err != nil {
-			glog.Warningf("status: %v", err)
-			return false, nil
-		}
-		if status != state.Running {
-			return false, nil
-		}
-		return true, nil
-	}
-
-	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil {
-		return fmt.Errorf("apiserver healthz never reported healthy")
-	}
-
-	vcheck := func() (bool, error) {
-		if time.Since(start) > timeout {
-			return false, fmt.Errorf("cluster wait timed out during version check")
-		}
-		if err := APIServerVersionMatch(client, cfg.KubernetesConfig.KubernetesVersion); err != nil {
-			glog.Warningf("api server version match failed: %v", err)
-			return false, nil
-		}
-		return true, nil
-	}
-
-	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, vcheck); err != nil {
-		return fmt.Errorf("controlPlane never updated to %s", cfg.KubernetesConfig.KubernetesVersion)
-	}
-
-	glog.Infof("duration metric: took %s to wait for apiserver health ...", time.Since(hStart))
-	return nil
-}
-
-// APIServerVersionMatch checks if the server version matches the expected
-func APIServerVersionMatch(client *kubernetes.Clientset, expected string) error {
-	vi, err := client.ServerVersion()
-	if err != nil {
-		return errors.Wrap(err, "server version")
-	}
-	glog.Infof("control plane version: %s", vi)
-	if version.CompareKubeAwareVersionStrings(vi.String(), expected) != 0 {
-		return fmt.Errorf("controlPane = %q, expected: %q", vi.String(), expected)
-	}
-	return nil
+// DontWait will return true if the config is no need to wait
+func DontWait(wcs map[string]bool) bool {
+	return wcs[APIServerWait] == false && wcs[SystemPodsWait] == false && wcs[DefaultServiceAccountWait] == false
 }
 
 // announceProblems checks for problems, and slows polling down if any are found
@@ -189,142 +94,6 @@ func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg conf
 	}
 }
 
-<<<<<<< HEAD
-// APIServerStatus returns apiserver status in libmachine style state.State
-func APIServerStatus(cr command.Runner, hostname string, port int) (state.State, error) {
-	glog.Infof("Checking apiserver status ...")
-
-	pid, err := apiServerPID(cr)
-	if err != nil {
-		glog.Warningf("stopped: unable to get apiserver pid: %v", err)
-		return state.Stopped, nil
-	}
-
-	// Get the freezer cgroup entry for this pid
-	rr, err := cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", fmt.Sprintf("/proc/%d/cgroup", pid)))
-	if err != nil {
-		glog.Warningf("unable to find freezer cgroup: %v", err)
-		return apiServerHealthz(hostname, port)
-
-	}
-	freezer := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("apiserver freezer: %q", freezer)
-	fparts := strings.Split(freezer, ":")
-	if len(fparts) != 3 {
-		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
-		return apiServerHealthz(hostname, port)
-	}
-
-	rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
-	if err != nil {
-		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
-		return apiServerHealthz(hostname, port)
-	}
-
-	fs := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("freezer state: %q", fs)
-	if fs == "FREEZING" || fs == "FROZEN" {
-		return state.Paused, nil
-	}
-	return apiServerHealthz(hostname, port)
-}
-
-// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State
-func apiServerHealthz(hostname string, port int) (state.State, error) {
-	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(hostname, fmt.Sprint(port)))
-	glog.Infof("Checking apiserver healthz at %s ...", url)
-	// To avoid: x509: certificate signed by unknown authority
-	tr := &http.Transport{
-		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	client := &http.Client{Transport: tr}
-	resp, err := client.Get(url)
-	// Connection refused, usually.
-	if err != nil {
-		glog.Infof("stopped: %s: %v", url, err)
-		return state.Stopped, nil
-	}
-	if resp.StatusCode == http.StatusUnauthorized {
-		glog.Errorf("%s returned code %d (unauthorized). Please ensure that your apiserver authorization settings make sense!", url, resp.StatusCode)
-		return state.Error, nil
-	}
-	if resp.StatusCode != http.StatusOK {
-		glog.Warningf("%s response: %v %+v", url, err, resp)
-		return state.Error, nil
-	}
-	return state.Running, nil
-}
-
-||||||| merged common ancestors
-// APIServerStatus returns apiserver status in libmachine style state.State
-func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error) {
-	glog.Infof("Checking apiserver status ...")
-
-	pid, err := apiServerPID(cr)
-	if err != nil {
-		glog.Warningf("stopped: unable to get apiserver pid: %v", err)
-		return state.Stopped, nil
-	}
-
-	// Get the freezer cgroup entry for this pid
-	rr, err := cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", fmt.Sprintf("/proc/%d/cgroup", pid)))
-	if err != nil {
-		glog.Warningf("unable to find freezer cgroup: %v", err)
-		return apiServerHealthz(ip, port)
-
-	}
-	freezer := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("apiserver freezer: %q", freezer)
-	fparts := strings.Split(freezer, ":")
-	if len(fparts) != 3 {
-		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
-		return apiServerHealthz(ip, port)
-	}
-
-	rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
-	if err != nil {
-		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
-		return apiServerHealthz(ip, port)
-	}
-
-	fs := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("freezer state: %q", fs)
-	if fs == "FREEZING" || fs == "FROZEN" {
-		return state.Paused, nil
-	}
-	return apiServerHealthz(ip, port)
-}
-
-// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State
-func apiServerHealthz(ip net.IP, port int) (state.State, error) {
-	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(port)))
-	glog.Infof("Checking apiserver healthz at %s ...", url)
-	// To avoid: x509: certificate signed by unknown authority
-	tr := &http.Transport{
-		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	client := &http.Client{Transport: tr}
-	resp, err := client.Get(url)
-	// Connection refused, usually.
-	if err != nil {
-		glog.Infof("stopped: %s: %v", url, err)
-		return state.Stopped, nil
-	}
-	if resp.StatusCode == http.StatusUnauthorized {
-		glog.Errorf("%s returned code %d (unauthorized). Please ensure that your apiserver authorization settings make sense!", url, resp.StatusCode)
-		return state.Error, nil
-	}
-	if resp.StatusCode != http.StatusOK {
-		glog.Warningf("%s response: %v %+v", url, err, resp)
-		return state.Error, nil
-	}
-	return state.Running, nil
-}
-
-=======
->>>>>>> break down kverify package to files
 // KubeletStatus checks the kubelet status
 func KubeletStatus(cr command.Runner) (state.State, error) {
 	glog.Infof("Checking kubelet status ...")
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 49881c61cc03..f0d9a0882b7f 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -345,7 +345,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		glog.Infof("%s is not a control plane, nothing to wait for", n.Name)
 		return nil
 	}
-	if cfg.WaitForCompos == kverify.NoWaitsCompos {
+	if kverify.DontWait(cfg.WaitForCompos) {
 		glog.Infof("skip waiting for components based on config.")
 		return nil
 	}
@@ -361,11 +361,15 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 	}
 
 	if cfg.WaitForCompos[kverify.APIServerWait] {
+		client, err := k.client(ip, port)
+		if err != nil {
+			return errors.Wrap(err, "get k8s client")
+		}
 		if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil {
 			return errors.Wrap(err, "wait for apiserver proc")
 		}
 
-		if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, start, ip, port, timeout); err != nil {
+		if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, start, ip, port, timeout); err != nil {
 			return errors.Wrap(err, "wait for healthy API server")
 		}
 	}

From d6b3cc6109f44684984666f47187b9ad3b972103 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 01:07:34 -0700
Subject: [PATCH 07/20] lint

---
 cmd/minikube/cmd/start.go                           | 5 +++--
 pkg/minikube/bootstrapper/bsutil/kverify/kverify.go | 2 +-
 pkg/minikube/node/start.go                          | 4 ++--
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 82112c1f2105..691d41f4b132 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -172,7 +172,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
-	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitsKeys, fmt.Sprintf("comma separated list of kuberentes components to wait to verify after start. defaults to %q, available options: %q . can also specify 'all' or 'none'", kverify.DefaultWaitsKeys, kverify.AllValidWaitsList))
+	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitsKeys, fmt.Sprintf("comma separated list of kuberentes components to verify and wait for after starting a cluster. defaults to %q, available options: %q . other acceptable values are 'all' or 'none', 'true' and 'false'", strings.Join(kverify.DefaultWaitsKeys, ","), strings.Join(kverify.AllValidWaitsList, ",")))
 	startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
 	startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
@@ -1069,6 +1069,7 @@ func createNode(cmd *cobra.Command, k8sVersion, kubeNodeName, drvName, repositor
 		},
 		Nodes: []config.Node{cp},
 	}
+	cfg.WaitForCompos = interpretWaitFlag(*cmd)
 	return cfg, cp, nil
 }
 
@@ -1238,7 +1239,7 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 			}
 		}
 		if !seen {
-			glog.Warning("invalid wait component flag %q. valid options are %q", wc, strings.Join(kverify.AllValidWaitsList, ","))
+			glog.Warningf("The value %q is invalid for --wait flag s! valid options are %q", wc, strings.Join(kverify.AllValidWaitsList, ","))
 		}
 	}
 	return waitCompos
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 3b6cba29882c..95e5d7735597 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -82,7 +82,7 @@ func podStatusMsg(pod core.Pod) string {
 
 // DontWait will return true if the config is no need to wait
 func DontWait(wcs map[string]bool) bool {
-	return wcs[APIServerWait] == false && wcs[SystemPodsWait] == false && wcs[DefaultServiceAccountWait] == false
+	return !wcs[APIServerWait] && !wcs[SystemPodsWait] && !wcs[DefaultServiceAccountWait]
 }
 
 // announceProblems checks for problems, and slows polling down if any are found
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 137cd669eecf..e0c1a9d7b1c8 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -35,6 +35,7 @@ import (
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/addons"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -56,7 +57,6 @@ import (
 
 const (
 	waitTimeout      = "wait-timeout"
-	waitUntilHealthy = "wait"
 	embedCerts       = "embed-certs"
 	keepContext      = "keep-context"
 	imageRepository  = "image-repository"
@@ -156,7 +156,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo
 		}
 
 		// Skip pre-existing, because we already waited for health
-		if !preExists {
+		if kverify.DontWait(cc.WaitForCompos) || !preExists {
 			if err := bs.WaitForNode(cc, n, viper.GetDuration(waitTimeout)); err != nil {
 				return nil, errors.Wrap(err, "Wait failed")
 			}

From 9fc36a6d4eecc7ffbcdf66b0b38f16b828c91942 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 01:10:39 -0700
Subject: [PATCH 08/20] typo

---
 cmd/minikube/cmd/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 691d41f4b132..98d959820229 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1239,7 +1239,7 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 			}
 		}
 		if !seen {
-			glog.Warningf("The value %q is invalid for --wait flag s! valid options are %q", wc, strings.Join(kverify.AllValidWaitsList, ","))
+			glog.Warningf("The value %q is invalid for --wait flag. valid options are %q", wc, strings.Join(kverify.AllValidWaitsList, ","))
 		}
 	}
 	return waitCompos

From 9335d28c43f183f6fed702b13925d517abfa5ec0 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 01:11:55 -0700
Subject: [PATCH 09/20] fix the years

---
 pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go  | 2 +-
 pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
index 239d01604aa0..214b097f732e 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors All rights reserved.
+Copyright 2020 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
index 700b8f2d8370..1efae9b19e01 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors All rights reserved.
+Copyright 2020 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

From e18ce70e0f96843c77c114455799c34ae0d6fb9a Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 01:13:24 -0700
Subject: [PATCH 10/20] fix negated logic

---
 pkg/minikube/node/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index e0c1a9d7b1c8..6bc29cb68b5c 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -156,7 +156,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo
 		}
 
 		// Skip pre-existing, because we already waited for health
-		if kverify.DontWait(cc.WaitForCompos) || !preExists {
+		if !kverify.DontWait(cc.WaitForCompos) && !preExists {
 			if err := bs.WaitForNode(cc, n, viper.GetDuration(waitTimeout)); err != nil {
 				return nil, errors.Wrap(err, "Wait failed")
 			}

From 89f2b9db6ee99f50c7f9d7dc09652f205f83ffa8 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 17:52:54 -0700
Subject: [PATCH 11/20] rename

---
 .../bootstrapper/bsutil/kverify/{kverify.go => kverify2.go}       | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename pkg/minikube/bootstrapper/bsutil/kverify/{kverify.go => kverify2.go} (100%)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify2.go
similarity index 100%
rename from pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
rename to pkg/minikube/bootstrapper/bsutil/kverify/kverify2.go

From 86c9d2b747575e917087c669759ae1d92ba146bb Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 17:53:20 -0700
Subject: [PATCH 12/20] rename

---
 pkg/minikube/bootstrapper/kubeadm/{kubeadm.go => kubeadm2.go} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename pkg/minikube/bootstrapper/kubeadm/{kubeadm.go => kubeadm2.go} (100%)

diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm2.go
similarity index 100%
rename from pkg/minikube/bootstrapper/kubeadm/kubeadm.go
rename to pkg/minikube/bootstrapper/kubeadm/kubeadm2.go

From 62ca22a66297b51e9007503becf35477d9c5fad3 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 18:58:51 -0700
Subject: [PATCH 13/20] refactor and rebase

---
 cmd/minikube/cmd/start.go                     |  20 ++--
 .../bootstrapper/bsutil/kverify/api_server.go | 106 +++++++++---------
 .../bootstrapper/bsutil/kverify/default_sa.go |   2 +-
 .../kverify/{kverify2.go => kverify.go}       |  92 +++++++++++----
 .../bsutil/kverify/system_pods.go             |  44 --------
 .../kubeadm/{kubeadm2.go => kubeadm.go}       |  22 ++--
 pkg/minikube/node/start.go                    |   2 +-
 7 files changed, 143 insertions(+), 145 deletions(-)
 rename pkg/minikube/bootstrapper/bsutil/kverify/{kverify2.go => kverify.go} (50%)
 rename pkg/minikube/bootstrapper/kubeadm/{kubeadm2.go => kubeadm.go} (98%)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 98d959820229..3a62c108098f 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -172,7 +172,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
-	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitsKeys, fmt.Sprintf("comma separated list of kuberentes components to verify and wait for after starting a cluster. defaults to %q, available options: %q . other acceptable values are 'all' or 'none', 'true' and 'false'", strings.Join(kverify.DefaultWaitsKeys, ","), strings.Join(kverify.AllValidWaitsList, ",")))
+	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitList, fmt.Sprintf("comma separated list of kuberentes components to verify and wait for after starting a cluster. defaults to %q, available options: %q . other acceptable values are 'all' or 'none', 'true' and 'false'", strings.Join(kverify.DefaultWaitList, ","), strings.Join(kverify.AllValidWaitList, ",")))
 	startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
 	startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
@@ -1207,31 +1207,31 @@ func getKubernetesVersion(old *config.ClusterConfig) string {
 // returns map of components to wait for
 func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 	if !cmd.Flags().Changed(waitComponents) {
-		glog.Infof("Wait Components : %+v", kverify.DefaultWaits)
-		return kverify.DefaultWaits
+		glog.Infof("Wait Components : %+v", kverify.DefaultWaitComponents)
+		return kverify.DefaultWaitComponents
 	}
 
 	waitFlags, err := cmd.Flags().GetStringSlice(waitComponents)
 	if err != nil {
-		glog.Infof("failed to get wait from flags, will use default wait components : %+v", kverify.DefaultWaits)
-		return kverify.DefaultWaits
+		glog.Infof("failed to get wait from flags, will use default wait components : %+v", kverify.DefaultWaitComponents)
+		return kverify.DefaultWaitComponents
 	}
 
 	// before minikube 1.9.0, wait flag was boolean
 	if (len(waitFlags) == 1 && waitFlags[0] == "true") || (len(waitFlags) == 1 && waitFlags[0] == "all") {
-		return kverify.AllWaitsCompos
+		return kverify.AllWaitComponents
 	}
 
 	// respecting legacy flag format --wait=false
 	// before minikube 1.9.0, wait flag was boolean
 	if (len(waitFlags) == 1 && waitFlags[0] == "false") || len(waitFlags) == 1 && waitFlags[0] == "none" {
-		return kverify.NoWaitsCompos
+		return kverify.NoWaitComponents
 	}
 
-	waitCompos := kverify.NoWaitsCompos
+	waitCompos := kverify.NoWaitComponents
 	for _, wc := range waitFlags {
 		seen := false
-		for _, valid := range kverify.AllValidWaitsList {
+		for _, valid := range kverify.AllValidWaitList {
 			if wc == valid {
 				waitCompos[wc] = true
 				seen = true
@@ -1239,7 +1239,7 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 			}
 		}
 		if !seen {
-			glog.Warningf("The value %q is invalid for --wait flag. valid options are %q", wc, strings.Join(kverify.AllValidWaitsList, ","))
+			glog.Warningf("The value %q is invalid for --wait flag. valid options are %q", wc, strings.Join(kverify.AllValidWaitList, ","))
 		}
 	}
 	return waitCompos
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
index ae1dcc9c46df..6aa6763a268b 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
@@ -41,8 +41,44 @@ import (
 	"k8s.io/minikube/pkg/minikube/cruntime"
 )
 
+// WaitForAPIServerProcess waits for api server to be healthy returns error if it doesn't
+func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, start time.Time, timeout time.Duration) error {
+	glog.Infof("waiting for apiserver process to appear ...")
+	err := wait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during process check")
+		}
+
+		if time.Since(start) > minLogCheckTime {
+			announceProblems(r, bs, cfg, cr)
+			time.Sleep(kconst.APICallRetryInterval * 5)
+		}
+
+		if _, ierr := apiServerPID(cr); ierr != nil {
+			return false, nil
+		}
+
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("apiserver process never appeared")
+	}
+	glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start))
+	return nil
+}
+
+// apiServerPID returns our best guess to the apiserver pid
+func apiServerPID(cr command.Runner) (int, error) {
+	rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "-xnf", "kube-apiserver.*minikube.*"))
+	if err != nil {
+		return 0, err
+	}
+	s := strings.TrimSpace(rr.Stdout.String())
+	return strconv.Atoi(s)
+}
+
 // WaitForHealthyAPIServer waits for api server status to be running
-func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, ip string, port int, timeout time.Duration) error {
+func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, hostname string, port int, timeout time.Duration) error {
 	glog.Infof("waiting for apiserver healthz status ...")
 	hStart := time.Now()
 
@@ -56,7 +92,7 @@ func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, c
 			time.Sleep(kconst.APICallRetryInterval * 5)
 		}
 
-		status, err := apiServerHealthz(net.ParseIP(ip), port)
+		status, err := apiServerHealthz(hostname, port)
 		if err != nil {
 			glog.Warningf("status: %v", err)
 			return false, nil
@@ -90,44 +126,21 @@ func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, c
 	return nil
 }
 
-// WaitForAPIServerProcess waits for api server to be healthy returns error if it doesn't
-func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, start time.Time, timeout time.Duration) error {
-	glog.Infof("waiting for apiserver process to appear ...")
-	err := wait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) {
-		if time.Since(start) > timeout {
-			return false, fmt.Errorf("cluster wait timed out during process check")
-		}
-
-		if time.Since(start) > minLogCheckTime {
-			announceProblems(r, bs, cfg, cr)
-			time.Sleep(kconst.APICallRetryInterval * 5)
-		}
-
-		if _, ierr := apiServerPID(cr); ierr != nil {
-			return false, nil
-		}
-
-		return true, nil
-	})
+// APIServerVersionMatch checks if the server version matches the expected
+func APIServerVersionMatch(client *kubernetes.Clientset, expected string) error {
+	vi, err := client.ServerVersion()
 	if err != nil {
-		return fmt.Errorf("apiserver process never appeared")
+		return errors.Wrap(err, "server version")
 	}
-	glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start))
-	return nil
-}
-
-// apiServerPID returns our best guess to the apiserver pid
-func apiServerPID(cr command.Runner) (int, error) {
-	rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "-xnf", "kube-apiserver.*minikube.*"))
-	if err != nil {
-		return 0, err
+	glog.Infof("control plane version: %s", vi)
+	if version.CompareKubeAwareVersionStrings(vi.String(), expected) != 0 {
+		return fmt.Errorf("controlPane = %q, expected: %q", vi.String(), expected)
 	}
-	s := strings.TrimSpace(rr.Stdout.String())
-	return strconv.Atoi(s)
+	return nil
 }
 
 // APIServerStatus returns apiserver status in libmachine style state.State
-func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error) {
+func APIServerStatus(cr command.Runner, hostname string, port int) (state.State, error) {
 	glog.Infof("Checking apiserver status ...")
 
 	pid, err := apiServerPID(cr)
@@ -140,7 +153,7 @@ func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error
 	rr, err := cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", fmt.Sprintf("/proc/%d/cgroup", pid)))
 	if err != nil {
 		glog.Warningf("unable to find freezer cgroup: %v", err)
-		return apiServerHealthz(ip, port)
+		return apiServerHealthz(hostname, port)
 
 	}
 	freezer := strings.TrimSpace(rr.Stdout.String())
@@ -148,13 +161,13 @@ func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error
 	fparts := strings.Split(freezer, ":")
 	if len(fparts) != 3 {
 		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
-		return apiServerHealthz(ip, port)
+		return apiServerHealthz(hostname, port)
 	}
 
 	rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
 	if err != nil {
 		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
-		return apiServerHealthz(ip, port)
+		return apiServerHealthz(hostname, port)
 	}
 
 	fs := strings.TrimSpace(rr.Stdout.String())
@@ -162,12 +175,12 @@ func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error
 	if fs == "FREEZING" || fs == "FROZEN" {
 		return state.Paused, nil
 	}
-	return apiServerHealthz(ip, port)
+	return apiServerHealthz(hostname, port)
 }
 
 // apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State
-func apiServerHealthz(ip net.IP, port int) (state.State, error) {
-	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(port)))
+func apiServerHealthz(hostname string, port int) (state.State, error) {
+	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(hostname, fmt.Sprint(port)))
 	glog.Infof("Checking apiserver healthz at %s ...", url)
 	// To avoid: x509: certificate signed by unknown authority
 	tr := &http.Transport{
@@ -191,16 +204,3 @@ func apiServerHealthz(ip net.IP, port int) (state.State, error) {
 	}
 	return state.Running, nil
 }
-
-// APIServerVersionMatch checks if the server version matches the expected
-func APIServerVersionMatch(client *kubernetes.Clientset, expected string) error {
-	vi, err := client.ServerVersion()
-	if err != nil {
-		return errors.Wrap(err, "server version")
-	}
-	glog.Infof("control plane version: %s", vi)
-	if version.CompareKubeAwareVersionStrings(vi.String(), expected) != 0 {
-		return fmt.Errorf("controlPane = %q, expected: %q", vi.String(), expected)
-	}
-	return nil
-}
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
index 214b097f732e..3b2105c7e53a 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
@@ -47,7 +47,7 @@ func WaitForDefaultSA(cs *kubernetes.Clientset) error {
 		}
 		return fmt.Errorf("couldn't find default service account")
 	}
-	if err := retry.Expo(saReady, 500*time.Millisecond, 30*time.Second); err != nil {
+	if err := retry.Expo(saReady, 500*time.Millisecond, 60*time.Second); err != nil {
 		return errors.Wrapf(err, "waited %s for SA", time.Since(pStart))
 	}
 
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify2.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
similarity index 50%
rename from pkg/minikube/bootstrapper/bsutil/kverify/kverify2.go
rename to pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 95e5d7735597..6f735e268e4c 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify2.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Kubernetes Authors All rights reserved.
+Copyright 2016 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -26,6 +26,8 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -34,31 +36,78 @@ import (
 	"k8s.io/minikube/pkg/minikube/logs"
 )
 
+// minLogCheckTime how long to wait before spamming error logs to console
+const minLogCheckTime = 60 * time.Second
+
 const (
-	// minLogCheckTime how long to wait before spamming error logs to console
-	minLogCheckTime = 30 * time.Second
-	// APIServerWait is the name used in the flags for k8s api server
-	APIServerWait = "apiserver"
-	// SystemPodsWait is the name used in the flags for pods in the kube system
-	SystemPodsWait = "system_pods"
-	// DefaultServiceAccountWait is the name used in the flags for default service account
-	DefaultServiceAccountWait = "default_sa"
+	// APIServerWaitKey is the name used in the flags for k8s api server
+	APIServerWaitKey = "apiserver"
+	// SystemPodsWaitKey is the name used in the flags for pods in the kube system
+	SystemPodsWaitKey = "system_pods"
+	// DefaultSAWaitKey is the name used in the flags for default service account
+	DefaultSAWaitKey = "default_sa"
 )
 
-// DefaultWaits is map of the the default components to wait for
-var DefaultWaits = map[string]bool{APIServerWait: true, SystemPodsWait: true}
+//  vars related to the --wait flag
+var (
+	// DefaultWaitComponents is map of the the default components to wait for
+	DefaultWaitComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
+	// NoWaitComponents is map of componets to wait for if specified 'none' or 'false'
+	NoWaitComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false}
+	// AllWaitComponents is map for waiting for all components.
+	AllWaitComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true}
+	// DefaultWaitList is list of all default components to wait for
+	DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey}
+	// AllValidWaitList list of all valid components to wait for
+	AllValidWaitList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey}
+)
 
-// DefaultWaitsKeys is list of all default components to wait for
-var DefaultWaitsKeys = []string{APIServerWait, SystemPodsWait}
+// ShouldWait will return true if the config says need to wait
+func ShouldWait(wcs map[string]bool) bool {
+	return wcs[APIServerWaitKey] || wcs[SystemPodsWaitKey] || wcs[DefaultSAWaitKey]
+}
+
+// ExpectedComponentsRunning returns whether or not all expected components are running
+func ExpectedComponentsRunning(cs *kubernetes.Clientset) error {
+	expected := []string{
+		"kube-dns", // coredns
+		"etcd",
+		"kube-apiserver",
+		"kube-controller-manager",
+		"kube-proxy",
+		"kube-scheduler",
+	}
 
-// NoWaitsCompos is map of componets to wait for if specified 'none' or 'false'
-var NoWaitsCompos = map[string]bool{APIServerWait: false, SystemPodsWait: false, DefaultServiceAccountWait: false}
+	found := map[string]bool{}
 
-// AllWaitsCompos is map for waiting for all components.
-var AllWaitsCompos = map[string]bool{APIServerWait: true, SystemPodsWait: true, DefaultServiceAccountWait: true}
+	pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, pod := range pods.Items {
+		glog.Infof("found pod: %s", podStatusMsg(pod))
+		if pod.Status.Phase != core.PodRunning {
+			continue
+		}
+		for k, v := range pod.ObjectMeta.Labels {
+			if k == "component" || k == "k8s-app" {
+				found[v] = true
+			}
+		}
+	}
 
-// AllValidWaitsList list of all valid components to wait for
-var AllValidWaitsList = []string{APIServerWait, SystemPodsWait, DefaultServiceAccountWait}
+	missing := []string{}
+	for _, e := range expected {
+		if !found[e] {
+			missing = append(missing, e)
+		}
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("missing components: %v", strings.Join(missing, ", "))
+	}
+	return nil
+}
 
 // podStatusMsg returns a human-readable pod status, for generating debug status
 func podStatusMsg(pod core.Pod) string {
@@ -80,11 +129,6 @@ func podStatusMsg(pod core.Pod) string {
 	return sb.String()
 }
 
-// DontWait will return true if the config is no need to wait
-func DontWait(wcs map[string]bool) bool {
-	return !wcs[APIServerWait] && !wcs[SystemPodsWait] && !wcs[DefaultServiceAccountWait]
-}
-
 // announceProblems checks for problems, and slows polling down if any are found
 func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner) {
 	problems := logs.FindProblems(r, bs, cfg, cr)
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
index 1efae9b19e01..9993044ae76a 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
@@ -19,11 +19,9 @@ package kverify
 
 import (
 	"fmt"
-	"strings"
 	"time"
 
 	"github.com/golang/glog"
-	core "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
@@ -70,45 +68,3 @@ func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg con
 	glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
 	return nil
 }
-
-// ExpectedComponentsRunning returns whether or not all expected components are running
-func ExpectedComponentsRunning(cs *kubernetes.Clientset) error {
-	expected := []string{
-		"kube-dns", // coredns
-		"etcd",
-		"kube-apiserver",
-		"kube-controller-manager",
-		"kube-proxy",
-		"kube-scheduler",
-	}
-
-	found := map[string]bool{}
-
-	pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{})
-	if err != nil {
-		return err
-	}
-
-	for _, pod := range pods.Items {
-		glog.Infof("found pod: %s", podStatusMsg(pod))
-		if pod.Status.Phase != core.PodRunning {
-			continue
-		}
-		for k, v := range pod.ObjectMeta.Labels {
-			if k == "component" || k == "k8s-app" {
-				found[v] = true
-			}
-		}
-	}
-
-	missing := []string{}
-	for _, e := range expected {
-		if !found[e] {
-			missing = append(missing, e)
-		}
-	}
-	if len(missing) > 0 {
-		return fmt.Errorf("missing components: %v", strings.Join(missing, ", "))
-	}
-	return nil
-}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm2.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
similarity index 98%
rename from pkg/minikube/bootstrapper/kubeadm/kubeadm2.go
rename to pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index f0d9a0882b7f..dcab2a879985 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm2.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -1,12 +1,9 @@
 /*
 Copyright 2016 The Kubernetes Authors All rights reserved.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -341,18 +338,19 @@ func (k *Bootstrapper) client(ip string, port int) (*kubernetes.Clientset, error
 // WaitForNode blocks until the node appears to be healthy
 func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, timeout time.Duration) error {
 	start := time.Now()
+
 	if !n.ControlPlane {
 		glog.Infof("%s is not a control plane, nothing to wait for", n.Name)
 		return nil
 	}
-	if kverify.DontWait(cfg.WaitForCompos) {
+	if !kverify.ShouldWait(cfg.WaitForCompos) {
 		glog.Infof("skip waiting for components based on config.")
 		return nil
 	}
 
 	cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c})
 	if err != nil {
-		return errors.Wrap(err, "new cruntime")
+		return err
 	}
 
 	hostname, _, port, err := driver.ControlPaneEndpoint(&cfg, &n, cfg.Driver)
@@ -360,8 +358,8 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		return errors.Wrap(err, "get control plane endpoint")
 	}
 
-	if cfg.WaitForCompos[kverify.APIServerWait] {
-		client, err := k.client(ip, port)
+	if cfg.WaitForCompos[kverify.APIServerWaitKey] {
+		client, err := k.client(hostname, port)
 		if err != nil {
 			return errors.Wrap(err, "get k8s client")
 		}
@@ -369,13 +367,13 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 			return errors.Wrap(err, "wait for apiserver proc")
 		}
 
-		if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, start, ip, port, timeout); err != nil {
+		if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, start, hostname, port, timeout); err != nil {
 			return errors.Wrap(err, "wait for healthy API server")
 		}
 	}
 
-	if cfg.WaitForCompos[kverify.SystemPodsWait] {
-		client, err := k.client(ip, port)
+	if cfg.WaitForCompos[kverify.SystemPodsWaitKey] {
+		client, err := k.client(hostname, port)
 		if err != nil {
 			return errors.Wrap(err, "get k8s client")
 		}
@@ -384,8 +382,8 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		}
 	}
 
-	if cfg.WaitForCompos[kverify.DefaultServiceAccountWait] {
-		client, err := k.client(ip, port)
+	if cfg.WaitForCompos[kverify.DefaultSAWaitKey] {
+		client, err := k.client(hostname, port)
 		if err != nil {
 			return errors.Wrap(err, "get k8s client")
 		}
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 6bc29cb68b5c..8bc2da45a1d8 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -156,7 +156,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo
 		}
 
 		// Skip pre-existing, because we already waited for health
-		if !kverify.DontWait(cc.WaitForCompos) && !preExists {
+		if kverify.ShouldWait(cc.WaitForCompos) && !preExists {
 			if err := bs.WaitForNode(cc, n, viper.GetDuration(waitTimeout)); err != nil {
 				return nil, errors.Wrap(err, "Wait failed")
 			}

From 5f6b547a1dc827e9be5f562ab8ceb9db0e2294d1 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 19:18:06 -0700
Subject: [PATCH 14/20] add time out for wait for default sa

---
 pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go | 8 ++++++--
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go           | 6 +++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
index 3b2105c7e53a..b5b9ca89a5c9 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
@@ -29,10 +29,14 @@ import (
 )
 
 // WaitForDefaultSA waits for the default service account to be created.
-func WaitForDefaultSA(cs *kubernetes.Clientset) error {
+func WaitForDefaultSA(cs *kubernetes.Clientset, timeout time.Duration) error {
 	glog.Info("waiting for default service account to be created ...")
-	pStart := time.Now()
+	start := time.Now()
 	saReady := func() error {
+		if time.Since(start) > timeout {
+			glog.Error("exceeded max time to wait to verify cluster during waiting for default service account. will skip waiting.")
+			return nil
+		}
 		// equivalent to manual check of 'kubectl --context profile get serviceaccount default'
 		sas, err := cs.CoreV1().ServiceAccounts("default").List(meta.ListOptions{})
 		if err != nil {
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index dcab2a879985..1d13c3edba5c 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -350,7 +350,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 
 	cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c})
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "create runtme-manager %s", cfg.KubernetesConfig.ContainerRuntime)
 	}
 
 	hostname, _, port, err := driver.ControlPaneEndpoint(&cfg, &n, cfg.Driver)
@@ -387,8 +387,8 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		if err != nil {
 			return errors.Wrap(err, "get k8s client")
 		}
-		if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, client, start, timeout); err != nil {
-			return errors.Wrap(err, "waiting for system pods")
+		if err := kverify.WaitForDefaultSA(client, timeout); err != nil {
+			return errors.Wrap(err, "waiting for default service account")
 		}
 	}
 	glog.Infof("duration metric: took %s to wait for : %+v ...", time.Since(start), cfg.WaitForCompos)

From 90c251d7e9daa6c900858a1935bbd99c5579ec0e Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 19:19:23 -0700
Subject: [PATCH 15/20] lint

---
 pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
index b5b9ca89a5c9..16d527e95b04 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
@@ -51,10 +51,10 @@ func WaitForDefaultSA(cs *kubernetes.Clientset, timeout time.Duration) error {
 		}
 		return fmt.Errorf("couldn't find default service account")
 	}
-	if err := retry.Expo(saReady, 500*time.Millisecond, 60*time.Second); err != nil {
-		return errors.Wrapf(err, "waited %s for SA", time.Since(pStart))
+	if err := retry.Expo(saReady, 500*time.Millisecond, 120*time.Second); err != nil {
+		return errors.Wrapf(err, "waited %s for SA", time.Since(start))
 	}
 
-	glog.Infof("duration metric: took %s for default service account to be created ...", time.Since(pStart))
+	glog.Infof("duration metric: took %s for default service account to be created ...", time.Since(start))
 	return nil
 }

From fa9d47ae18e46ea11b191ebce262966f3a7567de Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 19:21:30 -0700
Subject: [PATCH 16/20] reuse timeout

---
 pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
index 16d527e95b04..453edff7037b 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
@@ -33,10 +33,6 @@ func WaitForDefaultSA(cs *kubernetes.Clientset, timeout time.Duration) error {
 	glog.Info("waiting for default service account to be created ...")
 	start := time.Now()
 	saReady := func() error {
-		if time.Since(start) > timeout {
-			glog.Error("exceeded max time to wait to verify cluster during waiting for default service account. will skip waiting.")
-			return nil
-		}
 		// equivalent to manual check of 'kubectl --context profile get serviceaccount default'
 		sas, err := cs.CoreV1().ServiceAccounts("default").List(meta.ListOptions{})
 		if err != nil {
@@ -51,7 +47,7 @@ func WaitForDefaultSA(cs *kubernetes.Clientset, timeout time.Duration) error {
 		}
 		return fmt.Errorf("couldn't find default service account")
 	}
-	if err := retry.Expo(saReady, 500*time.Millisecond, 120*time.Second); err != nil {
+	if err := retry.Expo(saReady, 500*time.Millisecond, timeout); err != nil {
 		return errors.Wrapf(err, "waited %s for SA", time.Since(start))
 	}
 

From 74551841cce9e93821468a72f2723a9c78f11c83 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 2 Apr 2020 23:41:45 -0700
Subject: [PATCH 17/20] address review comments

---
 cmd/minikube/cmd/start.go                     | 38 +++++++++----------
 .../bootstrapper/bsutil/kverify/kverify.go    | 18 ++++-----
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  |  5 ++-
 3 files changed, 32 insertions(+), 29 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 3a62c108098f..681e59b269c0 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -172,7 +172,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
-	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitList, fmt.Sprintf("comma separated list of kuberentes components to verify and wait for after starting a cluster. defaults to %q, available options: %q . other acceptable values are 'all' or 'none', 'true' and 'false'", strings.Join(kverify.DefaultWaitList, ","), strings.Join(kverify.AllValidWaitList, ",")))
+	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitList, fmt.Sprintf("comma separated list of kubernetes components to verify and wait for after starting a cluster. defaults to %q, available options: %q . other acceptable values are 'all' or 'none', 'true' and 'false'", strings.Join(kverify.DefaultWaitList, ","), strings.Join(kverify.AllComponentsList, ",")))
 	startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
 	startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
@@ -1207,40 +1207,40 @@ func getKubernetesVersion(old *config.ClusterConfig) string {
 // returns map of components to wait for
 func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 	if !cmd.Flags().Changed(waitComponents) {
-		glog.Infof("Wait Components : %+v", kverify.DefaultWaitComponents)
-		return kverify.DefaultWaitComponents
+		glog.Infof("Wait Components : %+v", kverify.DefaultComponents)
+		return kverify.DefaultComponents
 	}
 
 	waitFlags, err := cmd.Flags().GetStringSlice(waitComponents)
 	if err != nil {
-		glog.Infof("failed to get wait from flags, will use default wait components : %+v", kverify.DefaultWaitComponents)
-		return kverify.DefaultWaitComponents
+		glog.Infof("failed to get wait from flags, will use default wait components : %+v", kverify.DefaultComponents)
+		return kverify.DefaultComponents
 	}
 
-	// before minikube 1.9.0, wait flag was boolean
-	if (len(waitFlags) == 1 && waitFlags[0] == "true") || (len(waitFlags) == 1 && waitFlags[0] == "all") {
-		return kverify.AllWaitComponents
-	}
-
-	// respecting legacy flag format --wait=false
-	// before minikube 1.9.0, wait flag was boolean
-	if (len(waitFlags) == 1 && waitFlags[0] == "false") || len(waitFlags) == 1 && waitFlags[0] == "none" {
-		return kverify.NoWaitComponents
+	if len(waitFlags) == 1 {
+		// respecting legacy flag before minikube 1.9.0, wait flag was boolean
+		if waitFlags[0] == "false" || waitFlags[0] == "none" {
+			return kverify.NoComponents
+		}
+		// respecting legacy flag before minikube 1.9.0, wait flag was boolean
+		if waitFlags[0] == "true" || waitFlags[0] == "all" {
+			return kverify.AllComponents
+		}
 	}
 
-	waitCompos := kverify.NoWaitComponents
+	waitComponents := kverify.NoComponents
 	for _, wc := range waitFlags {
 		seen := false
-		for _, valid := range kverify.AllValidWaitList {
+		for _, valid := range kverify.AllComponentsList {
 			if wc == valid {
-				waitCompos[wc] = true
+				waitComponents[wc] = true
 				seen = true
 				continue
 			}
 		}
 		if !seen {
-			glog.Warningf("The value %q is invalid for --wait flag. valid options are %q", wc, strings.Join(kverify.AllValidWaitList, ","))
+			glog.Warningf("The value %q is invalid for --wait flag. valid options are %q", wc, strings.Join(kverify.AllComponentsList, ","))
 		}
 	}
-	return waitCompos
+	return waitComponents
 }
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 6f735e268e4c..b6b9db934073 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors All rights reserved.
+Copyright 2020 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -50,16 +50,16 @@ const (
 
 //  vars related to the --wait flag
 var (
-	// DefaultWaitComponents is map of the the default components to wait for
-	DefaultWaitComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
+	// DefaultComponents is map of the the default components to wait for
+	DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
 	// NoWaitComponents is map of componets to wait for if specified 'none' or 'false'
-	NoWaitComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false}
-	// AllWaitComponents is map for waiting for all components.
-	AllWaitComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true}
-	// DefaultWaitList is list of all default components to wait for
+	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false}
+	// AllComponents is map for waiting for all components.
+	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true}
+	// DefaultWaitList is list of all default components to wait for. only names to be used for start flags.
 	DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey}
-	// AllValidWaitList list of all valid components to wait for
-	AllValidWaitList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey}
+	// AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags.
+	AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey}
 )
 
 // ShouldWait will return true if the config says need to wait
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 1d13c3edba5c..c16723fb5001 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -1,9 +1,12 @@
 /*
-Copyright 2016 The Kubernetes Authors All rights reserved.
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
+
     http://www.apache.org/licenses/LICENSE-2.0
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

From 4221e63e131d4552caa5cb6b20fb36b1184290d3 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 3 Apr 2020 18:43:37 -0700
Subject: [PATCH 18/20] add logs for wait components

---
 cmd/minikube/cmd/start.go                    |  7 +++++--
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 10 +++++-----
 pkg/minikube/config/types.go                 |  2 +-
 pkg/minikube/node/start.go                   |  2 +-
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 681e59b269c0..75ac54f82594 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1069,7 +1069,7 @@ func createNode(cmd *cobra.Command, k8sVersion, kubeNodeName, drvName, repositor
 		},
 		Nodes: []config.Node{cp},
 	}
-	cfg.WaitForCompos = interpretWaitFlag(*cmd)
+	cfg.VerifyComponents = interpretWaitFlag(*cmd)
 	return cfg, cp, nil
 }
 
@@ -1207,7 +1207,7 @@ func getKubernetesVersion(old *config.ClusterConfig) string {
 // returns map of components to wait for
 func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 	if !cmd.Flags().Changed(waitComponents) {
-		glog.Infof("Wait Components : %+v", kverify.DefaultComponents)
+		glog.Infof("Wait components to verify : %+v", kverify.DefaultComponents)
 		return kverify.DefaultComponents
 	}
 
@@ -1220,10 +1220,12 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 	if len(waitFlags) == 1 {
 		// respecting legacy flag before minikube 1.9.0, wait flag was boolean
 		if waitFlags[0] == "false" || waitFlags[0] == "none" {
+			glog.Infof("Waiting for no components: %+v", kverify.NoComponents)
 			return kverify.NoComponents
 		}
 		// respecting legacy flag before minikube 1.9.0, wait flag was boolean
 		if waitFlags[0] == "true" || waitFlags[0] == "all" {
+			glog.Infof("Waiting for all components: %+v", kverify.AllComponents)
 			return kverify.AllComponents
 		}
 	}
@@ -1242,5 +1244,6 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 			glog.Warningf("The value %q is invalid for --wait flag. valid options are %q", wc, strings.Join(kverify.AllComponentsList, ","))
 		}
 	}
+	glog.Infof("Waiting for components: %+v", waitComponents)
 	return waitComponents
 }
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index c16723fb5001..e8d76def913b 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -346,7 +346,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		glog.Infof("%s is not a control plane, nothing to wait for", n.Name)
 		return nil
 	}
-	if !kverify.ShouldWait(cfg.WaitForCompos) {
+	if !kverify.ShouldWait(cfg.VerifyComponents) {
 		glog.Infof("skip waiting for components based on config.")
 		return nil
 	}
@@ -361,7 +361,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		return errors.Wrap(err, "get control plane endpoint")
 	}
 
-	if cfg.WaitForCompos[kverify.APIServerWaitKey] {
+	if cfg.VerifyComponents[kverify.APIServerWaitKey] {
 		client, err := k.client(hostname, port)
 		if err != nil {
 			return errors.Wrap(err, "get k8s client")
@@ -375,7 +375,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		}
 	}
 
-	if cfg.WaitForCompos[kverify.SystemPodsWaitKey] {
+	if cfg.VerifyComponents[kverify.SystemPodsWaitKey] {
 		client, err := k.client(hostname, port)
 		if err != nil {
 			return errors.Wrap(err, "get k8s client")
@@ -385,7 +385,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		}
 	}
 
-	if cfg.WaitForCompos[kverify.DefaultSAWaitKey] {
+	if cfg.VerifyComponents[kverify.DefaultSAWaitKey] {
 		client, err := k.client(hostname, port)
 		if err != nil {
 			return errors.Wrap(err, "get k8s client")
@@ -394,7 +394,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 			return errors.Wrap(err, "waiting for default service account")
 		}
 	}
-	glog.Infof("duration metric: took %s to wait for : %+v ...", time.Since(start), cfg.WaitForCompos)
+	glog.Infof("duration metric: took %s to wait for : %+v ...", time.Since(start), cfg.VerifyComponents)
 	return nil
 }
 
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index ef3b2d41cc01..95991d5f336f 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -65,7 +65,7 @@ type ClusterConfig struct {
 	KubernetesConfig        KubernetesConfig
 	Nodes                   []Node
 	Addons                  map[string]bool
-	WaitForCompos           map[string]bool // map of components to verify and wait for after start.
+	VerifyComponents        map[string]bool // map of components to verify and wait for after start.
 }
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 8bc2da45a1d8..e43e464d43d8 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -156,7 +156,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo
 		}
 
 		// Skip pre-existing, because we already waited for health
-		if kverify.ShouldWait(cc.WaitForCompos) && !preExists {
+		if kverify.ShouldWait(cc.VerifyComponents) && !preExists {
 			if err := bs.WaitForNode(cc, n, viper.GetDuration(waitTimeout)); err != nil {
 				return nil, errors.Wrap(err, "Wait failed")
 			}

From 478d1833314e38882f13760c992e51f2e9e3b17e Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 3 Apr 2020 18:46:18 -0700
Subject: [PATCH 19/20] use all components list in should wait

---
 pkg/minikube/bootstrapper/bsutil/kverify/kverify.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index b6b9db934073..15b89c6cdc6e 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -64,7 +64,12 @@ var (
 
 // ShouldWait will return true if the config says need to wait
 func ShouldWait(wcs map[string]bool) bool {
-	return wcs[APIServerWaitKey] || wcs[SystemPodsWaitKey] || wcs[DefaultSAWaitKey]
+	for _, c := range AllComponentsList {
+		if wcs[c] {
+			return true
+		}
+	}
+	return false
 }
 
 // ExpectedComponentsRunning returns whether or not all expected components are running

From 77d1da04a2b45fdae15243d346b981b1d0b4f38a Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 3 Apr 2020 18:48:39 -0700
Subject: [PATCH 20/20] change log level for err

---
 cmd/minikube/cmd/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 75ac54f82594..0a42f5e979b1 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1213,7 +1213,7 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool {
 
 	waitFlags, err := cmd.Flags().GetStringSlice(waitComponents)
 	if err != nil {
-		glog.Infof("failed to get wait from flags, will use default wait components : %+v", kverify.DefaultComponents)
+		glog.Warningf("Failed to read --wait from flags: %v.\n Moving on will use the default wait components: %+v", err, kverify.DefaultComponents)
 		return kverify.DefaultComponents
 	}