create all statefulsets immediately

Makefile

@@ -34,7 +34,7 @@ docker-push: docker backup-docker
 docker: build
 	docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator
 
-build: controller-manager scheduler discovery admission-controller
+build: controller-manager scheduler discovery admission-controller wait-for-pd
 
 controller-manager:
 	$(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-controller-manager cmd/controller-manager/main.go
@@ -48,6 +48,9 @@ discovery:
 admission-controller:
 	$(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-admission-controller cmd/admission-controller/main.go
 
+wait-for-pd:
+	$(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/wait-for-pd cmd/wait-for-pd/main.go
+
 backup-manager:
 	$(GO) -ldflags '$(LDFLAGS)' -o images/backup-manager/bin/tidb-backup-manager cmd/backup-manager/main.go
 

charts/tidb-operator/templates/controller-manager-deployment.yaml

@@ -56,6 +56,8 @@ spec:
                 fieldPath: metadata.namespace
           - name: TZ
             value: {{ .Values.timezone | default "UTC" }}
+          - name: MY_IMAGE
+            value: "{{ .Values.operatorImage }}"
     {{- with .Values.controllerManager.nodeSelector }}
       nodeSelector:
 {{ toYaml . | indent 8 }}

cmd/controller-manager/main.go

@@ -105,6 +105,10 @@ func main() {
 	if err != nil {
 		glog.Fatalf("failed to get kubernetes Clientset: %v", err)
 	}
+	operatorImage := os.Getenv("MY_IMAGE")
+	if operatorImage == "" {
+		glog.Fatal("MY_IMAGE env variable not set")
+	}
 
 	var informerFactory informers.SharedInformerFactory
 	var kubeInformerFactory kubeinformers.SharedInformerFactory
@@ -135,7 +139,7 @@ func main() {
 		},
 	}
 
-	tcController := tidbcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory, autoFailover, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod)
+	tcController := tidbcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory, autoFailover, operatorImage, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod)
 	backupController := backup.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
 	restoreController := restore.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
 	bsController := backupschedule.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)

cmd/wait-for-pd/main.go

@@ -0,0 +1,142 @@
+// Copyright 2019. PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/pingcap/tidb-operator/pkg/pdapi"
+	"github.com/pingcap/tidb-operator/pkg/version"
+	"k8s.io/apiserver/pkg/util/logs"
+)
+
+const (
+	timeout = 5 * time.Second
+)
+
+var (
+	printVersion          bool
+	waitForInitialization bool
+	waitForLeader         bool
+)
+
+func init() {
+	flag.BoolVar(&printVersion, "V", false, "Show version and quit")
+	flag.BoolVar(&printVersion, "version", false, "Show version and quit")
+	flag.BoolVar(&waitForInitialization, pdapi.WaitForInitializationFlag, false, "Wait for initialization of the cluster. This means all replicas have come online")
+	flag.BoolVar(&waitForLeader, pdapi.WaitForLeaderFlag, false, "Wait for just the presence of a PD leader.")
+	flag.Parse()
+}
+
+func pdHasLeader(pdClient pdapi.PDClient) (bool, error) {
+	memberInfo, err := pdClient.GetPDLeader()
+	if err != nil {
+		return false, err
+	}
+	return memberInfo != nil, nil
+}
+
+// On older versions this will return the empty semver version 0.0.0
+// Most tidb-operator users will be using version 3.0+ which has the version field.
+func pdVersion(pdClient pdapi.PDClient) (string, error) {
+	conf, err := pdClient.GetConfig()
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%s", conf.ClusterVersion), nil
+}
+
+// wait-for-pd waits for 1 PD to be running
+func main() {
+	if printVersion {
+		version.PrintVersionInfo()
+		os.Exit(0)
+	}
+	version.LogVersionInfo()
+
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	oneSecond, err := time.ParseDuration("1s")
+	if err != nil {
+		glog.Fatalf("Error parsing time %v", err)
+	}
+
+	tcName := os.Getenv("CLUSTER_NAME")
+	if tcName == "" {
+		glog.Fatalf("Expected CLUSTER_NAME env variable to be set")
+	}
+	namespace := os.Getenv("NAMESPACE")
+	if namespace == "" {
+		glog.Fatalf("Expected NAMESPACE env variable to be set")
+	}
+
+	// This is just a check for intialization at startup, it doesn't seem valuable to try to hide it with https
+	// Just being able to observe that there is network traffic going on with this program pretty much tells you everything you would want to know here.
+	pdClient := pdapi.NewPDClient(pdapi.PdClientURL(pdapi.Namespace(namespace), tcName, "http"), timeout, false)
+
+	var waitFunction func() bool
+
+	waitForLeaderFunc := func() bool {
+		hasLeader, err := pdHasLeader(pdClient)
+		if err != nil {
+			glog.Infof("Error using pdClient to get members %v", err)
+		} else if hasLeader {
+			glog.Infof("Found a PD member. Exiting now.")
+			return true
+		} else {
+			glog.Infof("PD Leader not found")
+		}
+		return false
+	}
+
+	waitForInitializationFunc := func() bool {
+		isInit, err := pdClient.GetClusterInitialized()
+		if err != nil {
+			glog.Infof("Error using pdClient to get cluster status %v", err)
+		} else if isInit == nil {
+			version, verr := pdVersion(pdClient)
+			if verr != nil || version == "" {
+				glog.Errorf("Error using pdClient to get cluster version %v", verr)
+			}
+			glog.Warningf("For this PD version %s the cluster status API does not support is_initialized. Will now wait for just a PD leader", version)
+			waitFunction = waitForLeaderFunc
+		} else if *isInit {
+			glog.Infof("Cluster is inititialized. Exiting now.")
+			return true
+		} else {
+			glog.Infof("PD is not initialized")
+		}
+		return false
+	}
+
+	if waitForInitialization {
+		waitFunction = waitForInitializationFunc
+	} else if waitForLeader {
+		waitFunction = waitForLeaderFunc
+	} else {
+		glog.Fatalf("Expected either the flag --initialization or --leader")
+	}
+
+	for {
+		if waitFunction() {
+			break
+		}
+		time.Sleep(oneSecond)
+	}
+}

go.mod

@@ -2,20 +2,15 @@ module github.com/pingcap/tidb-operator
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect
 	github.com/Microsoft/go-winio v0.4.12 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
-	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 // indirect
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/coreos/bbolt v1.3.1-coreos.6 // indirect
 	github.com/coreos/etcd v0.0.0-20180530235116-2b3aa7e1d49d // indirect
-	github.com/coreos/go-semver v0.2.0 // indirect
-	github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 // indirect
 	github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea // indirect
 	github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/dnephin/govet v0.0.0-20171012192244-4a96d43e39d3
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v0.7.3-0.20171023200535-7848b8beb9d3
@@ -24,36 +19,25 @@ require (
 	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
 	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/emicklei/go-restful v2.9.5+incompatible
 	github.com/evanphx/json-patch v4.1.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/fatih/camelcase v1.0.0 // indirect
-	github.com/fsnotify/fsnotify v1.4.7 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-sql-driver/mysql v1.4.0
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450 // indirect
 	github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995 // indirect
 	github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e // indirect
-	github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a // indirect
 	github.com/googleapis/gnostic v0.2.0 // indirect
 	github.com/gophercloud/gophercloud v0.3.0 // indirect
 	github.com/gorilla/context v1.1.1 // indirect
 	github.com/gorilla/mux v1.6.2 // indirect
-	github.com/gorilla/websocket v1.2.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect
-	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.4.1 // indirect
 	github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 // indirect
-	github.com/hpcloud/tail v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jonboulle/clockwork v0.1.0 // indirect
 	github.com/juju/errors v0.0.0-20180806074554-22422dad46e1
 	github.com/juju/loggo v0.0.0-20180524022052-584905176618 // indirect
 	github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 // indirect
@@ -63,20 +47,17 @@ require (
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/nwaples/rardecode v1.0.0 // indirect
-	github.com/onsi/ginkgo v1.6.0 // indirect
-	github.com/onsi/gomega v1.4.1
+	github.com/onsi/gomega v1.4.2
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pborman/uuid v1.2.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
-	github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 // indirect
 	github.com/pingcap/errors v0.11.0
-	github.com/pingcap/kvproto v0.0.0-20180606093822-b7ba8ea1c0b4
-	github.com/pingcap/pd v2.1.0-beta+incompatible
+	github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c
+	github.com/pingcap/pd v0.0.0-20190628112929-bfbaa0662040
 	github.com/pingcap/tidb v2.1.0-beta+incompatible
-	github.com/pkg/errors v0.8.0 // indirect
 	github.com/prometheus/client_golang v0.8.0
 	github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect
 	github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e // indirect
@@ -85,35 +66,20 @@ require (
 	github.com/robfig/cron v1.1.0
 	github.com/russross/blackfriday v1.5.2+incompatible // indirect
 	github.com/sirupsen/logrus v1.0.6
-	github.com/soheilhy/cmux v0.1.4 // indirect
 	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3
-	github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-client-go v2.16.0+incompatible // indirect
 	github.com/uber/jaeger-lib v2.0.0+incompatible // indirect
-	github.com/ugorji/go v1.1.1 // indirect
 	github.com/unrolled/render v0.0.0-20180807193321-4206df6ff701 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
-	github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect
 	github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect
-	go.uber.org/atomic v1.3.2 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
-	go.uber.org/zap v1.9.1 // indirect
 	golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect
-	google.golang.org/genproto v0.0.0-20180731170733-daca94659cb5 // indirect
-	google.golang.org/grpc v1.12.0 // indirect
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/fsnotify.v1 v1.4.7 // indirect
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3 // indirect
 	gopkg.in/square/go-jose.v2 v2.3.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0-20181128191700-6db15a15d2d3
 	k8s.io/apiextensions-apiserver v0.0.0-20190118124337-a384d17938fe // indirect