Merge pull request #1264 from hongchaodeng/sh

SelfHosted: increase timeout to wait pod running
coreos · Jul 7, 2017 · 9fabd46 · 9fabd46
2 parents 623d16b + e15a583
commit 9fabd46
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 4 deletions.
diff --git a/pkg/cluster/self_hosted.go b/pkg/cluster/self_hosted.go
@@ -154,7 +154,7 @@ func (c *Cluster) newSelfHostedSeedMember() error {
 	initialCluster := []string{newMember.Name + "=" + newMember.PeerURL()}
 
 	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, nil, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner())
-	_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
+	_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 3*60*time.Second)
 	if err != nil {
 		return err
 	}
@@ -191,7 +191,7 @@ func (c *Cluster) migrateBootMember() error {
 
 	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, []string{endpoint}, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
 	ns := c.cluster.Metadata.Namespace
-	_, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 30*time.Second)
+	_, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 3*60*time.Second)
 	if err != nil {
 		return err
 	}

diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go
@@ -145,7 +145,7 @@ func CreateAndWaitPod(kubecli kubernetes.Interface, ns string, pod *v1.Pod, time
 		return nil, err
 	}
 
-	interval := 3 * time.Second
+	interval := 5 * time.Second
 	var retPod *v1.Pod
 	err = retryutil.Retry(interval, int(timeout/(interval)), func() (bool, error) {
 		retPod, err = kubecli.CoreV1().Pods(ns).Get(pod.Name, metav1.GetOptions{})
@@ -162,7 +162,14 @@ func CreateAndWaitPod(kubecli kubernetes.Interface, ns string, pod *v1.Pod, time
 		}
 	})
 
-	return retPod, err
+	if err != nil {
+		if retryutil.IsRetryFailure(err) {
+			return nil, fmt.Errorf("failed to wait pod running, it is still pending: %v", err)
+		}
+		return nil, fmt.Errorf("failed to wait pod running: %v", err)
+	}
+
+	return retPod, nil
 }
 
 func newEtcdServiceManifest(svcName, clusterName string, clusterIP string, port int32) *v1.Service {