Merge pull request #29 from cbws/quorum-lost-failed

Mark cluster status failed when no pods are running or quorum is lost

pkg/cluster/cluster.go

@@ -41,6 +41,7 @@ import (
 var (
 	reconcileInterval         = 8 * time.Second
 	podTerminationGracePeriod = int64(5)
+	errAllPodsDead = newFatalError("all etcd pods are dead.")
 )
 
 type clusterEventType string
@@ -246,8 +247,8 @@ func (c *Cluster) run() {
 				continue
 			}
 			if len(running) == 0 {
-				// TODO: how to handle this case?
-				c.logger.Warningf("all etcd pods are dead.")
+				// All pods are dead, cluster cannot recover so return error
+				rerr = errAllPodsDead
 				break
 			}
 

pkg/cluster/reconcile.go

@@ -16,7 +16,6 @@ package cluster
 
 import (
 	"context"
-	"errors"
 	"fmt"
 
 	api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2"
@@ -30,7 +29,7 @@ import (
 )
 
 // ErrLostQuorum indicates that the etcd cluster lost its quorum.
-var ErrLostQuorum = errors.New("lost quorum")
+var ErrLostQuorum = newFatalError("lost quorum")
 
 // reconcile reconciles cluster current state to desired state specified by spec.
 // - it tries to reconcile the cluster to desired size.