Merge pull request #1165 from hasbro17/haseeb/operator-readiness-probe

*: add readiness probe for operator pod

cmd/operator/main.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 	"os/signal"
 	"runtime"
@@ -32,6 +33,7 @@ import (
 	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
 	"github.com/coreos/etcd-operator/pkg/util/k8sutil/election"
 	"github.com/coreos/etcd-operator/pkg/util/k8sutil/election/resourcelock"
+	"github.com/coreos/etcd-operator/pkg/util/probe"
 	"github.com/coreos/etcd-operator/pkg/util/retryutil"
 	"github.com/coreos/etcd-operator/version"
 
@@ -54,6 +56,7 @@ var (
 	awsSecret        string
 	awsConfig        string
 	s3Bucket         string
+	listenAddr       string
 	gcInterval       time.Duration
 
 	chaosLevel int
@@ -76,6 +79,7 @@ func init() {
 	flag.StringVar(&awsConfig, "backup-aws-config", "",
 		"The name of the kube configmap object that stores the AWS config file. The file name must be 'config'.")
 	flag.StringVar(&s3Bucket, "backup-s3-bucket", "", "The name of the AWS S3 bucket to store backups in.")
+	flag.StringVar(&listenAddr, "listen-addr", "0.0.0.0:8080", "The address on which the HTTP server will listen to")
 	// chaos level will be removed once we have a formal tool to inject failures.
 	flag.IntVar(&chaosLevel, "chaos-level", -1, "DO NOT USE IN PRODUCTION - level of chaos injected into the etcd clusters created by the operator.")
 	flag.BoolVar(&printVersion, "version", false, "Show version and quit")
@@ -151,6 +155,9 @@ func main() {
 		},
 	}
 
+	http.HandleFunc(probe.HTTPReadyzEndpoint, probe.ReadyzHandler)
+	go http.ListenAndServe(listenAddr, nil)
+
 	election.RunOrDie(election.LeaderElectionConfig{
 		Lock:          rl,
 		LeaseDuration: leaseDuration,

pkg/controller/controller.go

@@ -29,6 +29,7 @@ import (
 	"github.com/coreos/etcd-operator/pkg/spec"
 	"github.com/coreos/etcd-operator/pkg/util/constants"
 	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
+	"github.com/coreos/etcd-operator/pkg/util/probe"
 
 	"github.com/Sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -314,6 +315,8 @@ func (c *Controller) watch(watchVersion string) (<-chan *Event, <-chan error) {
 
 			c.logger.Infof("start watching at %v", watchVersion)
 
+			probe.SetReady()
+
 			decoder := json.NewDecoder(resp.Body)
 			for {
 				ev, st, err := pollEvent(decoder)

pkg/util/probe/readyz.go

@@ -0,0 +1,47 @@
+// Copyright 2017 The etcd-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package probe
+
+import (
+	"net/http"
+	"sync"
+)
+
+const (
+	HTTPReadyzEndpoint = "/readyz"
+)
+
+var (
+	mu    sync.Mutex
+	ready = false
+)
+
+func SetReady() {
+	mu.Lock()
+	ready = true
+	mu.Unlock()
+}
+
+// ReadyzHandler writes back the HTTP status code 200 if the operator is ready, and 500 otherwise
+func ReadyzHandler(w http.ResponseWriter, r *http.Request) {
+	mu.Lock()
+	isReady := ready
+	mu.Unlock()
+	if isReady {
+		w.WriteHeader(http.StatusOK)
+	} else {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}