*: rebase release 3.0 to master

Makefile

@@ -12,8 +12,8 @@ GOCHECKER := awk '{ print } END { if (NR > 0) { exit 1 } }'
 RETOOL := ./scripts/retool
 OVERALLS := overalls
 
-GOFAIL_ENABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|\.retools)" | xargs ./scripts/retool do gofail enable)
-GOFAIL_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|\.retools)" | xargs ./scripts/retool do gofail disable)
+FAILPOINT_ENABLE  := $$(find $$PWD/ -type d | grep -vE "(\.git|\.retools)" | xargs ./scripts/retool do failpoint-ctl enable)
+FAILPOINT_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|\.retools)" | xargs ./scripts/retool do failpoint-ctl disable)
 
 LDFLAGS += -X "$(PD_PKG)/server.PDReleaseVersion=$(shell git describe --tags --dirty)"
 LDFLAGS += -X "$(PD_PKG)/server.PDBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
@@ -48,14 +48,14 @@ endif
 
 test: retool-setup
 	# testing..
-	@$(GOFAIL_ENABLE)
-	CGO_ENABLED=1 GO111MODULE=on go test -race -cover $(TEST_PKGS) || { $(GOFAIL_DISABLE); exit 1; }
-	@$(GOFAIL_DISABLE)
+	@$(FAILPOINT_ENABLE)
+	CGO_ENABLED=1 GO111MODULE=on go test -race -cover $(TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
 
 basic-test:
-	@$(GOFAIL_ENABLE)
-	GO111MODULE=on go test $(BASIC_TEST_PKGS) || { $(GOFAIL_DISABLE); exit 1; }
-	@$(GOFAIL_DISABLE)
+	@$(FAILPOINT_ENABLE)
+	GO111MODULE=on go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
 
 # These need to be fixed before they can be ran regularly
 check-fail:
@@ -98,9 +98,9 @@ tidy:
 travis_coverage: export GO111MODULE=on
 travis_coverage:
 ifeq ("$(TRAVIS_COVERAGE)", "1")
-	@$(GOFAIL_ENABLE)
-	CGO_ENABLED=1 ./scripts/retool do $(OVERALLS) -project=github.com/pingcap/pd -covermode=count -ignore='.git,vendor' -- -coverpkg=./... || { $(GOFAIL_DISABLE); exit 1; }
-	@$(GOFAIL_DISABLE)
+	@$(FAILPOINT_ENABLE)
+	CGO_ENABLED=1 ./scripts/retool do $(OVERALLS) -project=github.com/pingcap/pd -covermode=count -ignore='.git,vendor' -- -coverpkg=./... || { $(FAILPOINT_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
 else
 	@echo "coverage only runs in travis."
 endif
@@ -114,12 +114,12 @@ clean-test:
 	rm -rf /tmp/pd-tests*
 	rm -rf /tmp/test_etcd*
 
-gofail-enable:
-	# Converting gofail failpoints...
-	@$(GOFAIL_ENABLE)
+failpoint-enable:
+	# Converting failpoints...
+	@$(FAILPOINT_ENABLE)
 
-gofail-disable:
-	# Restoring gofail failpoints...
-	@$(GOFAIL_DISABLE)
+failpoint-disable:
+	# Restoring failpoints...
+	@$(FAILPOINT_DISABLE)
 
 .PHONY: all ci vendor clean-test tidy

README.md

@@ -1,14 +1,14 @@
-# PD 
+# PD
 
 [![TravisCI Build Status](https://travis-ci.org/pingcap/pd.svg?branch=master)](https://travis-ci.org/pingcap/pd)
 ![GitHub release](https://img.shields.io/github/release/pingcap/pd.svg)
 [![CircleCI Build Status](https://circleci.com/gh/pingcap/pd.svg?style=shield)](https://circleci.com/gh/pingcap/pd)
 [![Go Report Card](https://goreportcard.com/badge/github.com/pingcap/pd)](https://goreportcard.com/report/github.com/pingcap/pd)
 [![codecov](https://codecov.io/gh/pingcap/pd/branch/master/graph/badge.svg)](https://codecov.io/gh/pingcap/pd)
 
-PD is the abbreviation for Placement Driver. It is used to manage and schedule the [TiKV](https://github.com/tikv/tikv) cluster. 
+PD is the abbreviation for Placement Driver. It is used to manage and schedule the [TiKV](https://github.com/tikv/tikv) cluster.
 
-PD supports distribution and fault-tolerance by embedding [etcd](https://github.com/etcd-io/etcd). 
+PD supports distribution and fault-tolerance by embedding [etcd](https://github.com/etcd-io/etcd).
 
 ## Build
 
@@ -19,15 +19,15 @@ PD supports distribution and fault-tolerance by embedding [etcd](https://github.
 
 ### Command flags
 
-See [configuration](https://github.com/pingcap/docs/blob/master/op-guide/configuration.md#placement-driver-pd).
+See [PD Configuration Flags](https://pingcap.com/docs/dev/reference/configuration/pd-server/configuration/#pd-configuration-flags).
 
 ### Single Node with default ports
 
-You can run `pd-server` directly on your local machine, if you want to connect to PD from outside, 
+You can run `pd-server` directly on your local machine, if you want to connect to PD from outside,
 you can let PD listen on the host IP.
 
 ```bash
-# Set correct HostIP here. 
+# Set correct HostIP here.
 export HostIP="192.168.199.105"
 
 pd-server --name="pd" \
@@ -60,9 +60,9 @@ X-Etcd-Cluster-Id: 33dc747581249309
         {
             "clientURLs": [
                 "http://192.168.199.105:2379"
-            ], 
-            "id": "f62e88a6e81c149", 
-            "name": "pd", 
+            ],
+            "id": "f62e88a6e81c149",
+            "name": "pd",
             "peerURLs": [
                 "http://192.168.199.105:2380"
             ]
@@ -75,20 +75,20 @@ X-Etcd-Cluster-Id: 33dc747581249309
 
 You can use the following command to build a PD image directly:
 
-```
+```bash
 docker build -t pingcap/pd .
 ```
 
 Or you can also use following command to get PD from Docker hub:
 
-```
+```bash
 docker pull pingcap/pd
 ```
 
-Run a single node with Docker: 
+Run a single node with Docker:
 
 ```bash
-# Set correct HostIP here. 
+# Set correct HostIP here.
 export HostIP="192.168.199.105"
 
 docker run -d -p 2379:2379 -p 2380:2380 --name pd pingcap/pd \
@@ -103,9 +103,9 @@ docker run -d -p 2379:2379 -p 2380:2380 --name pd pingcap/pd \
 
 ### Cluster
 
-PD is a component in TiDB project, you must run it with TiDB and TiKV together, see 
-[TiDB-Ansible](https://github.com/pingcap/docs/blob/master/op-guide/ansible-deployment.md) to learn 
-how to set up the cluster and run them.
+PD is a component in TiDB project, you must run it with TiDB and TiKV together, see
+[TiDB-Ansible](https://pingcap.com/docs/dev/how-to/deploy/orchestrated/ansible/#deploy-tidb-using-ansible)
+to learn how to set up the cluster and run them.
 
-You can also use [Docker](https://github.com/pingcap/docs/blob/master/op-guide/docker-deployment.md) to 
-run the cluster.
+You can also use [Docker](https://pingcap.com/docs/dev/how-to/deploy/orchestrated/docker/#deploy-tidb-using-docker)
+to run the cluster.

build.ps1

@@ -0,0 +1,16 @@
+# For `--version`
+$PD_PKG = "github.com/pingcap/pd"
+$GO_LDFLAGS = "-X `"$PD_PKG/server.PDReleaseVersion=$(git describe --tags --dirty)`""
+$GO_LDFLAGS += " -X `"$PD_PKG/server.PDBuildTS=$(date -u '+%Y-%m-%d_%I:%M:%S')`""
+$GO_LDFLAGS += " -X `"$PD_PKG/server.PDGitHash=$(git rev-parse HEAD)`""
+$GO_LDFLAGS += " -X `"$PD_PKG/server.PDGitBranch=$(git rev-parse --abbrev-ref HEAD)`""
+
+# Output binaries
+go build -ldflags $GO_LDFLAGS -o bin/pd-server.exe cmd/pd-server/main.go
+echo "bin/pd-server.exe"
+go build -ldflags $GO_LDFLAGS -o bin/pd-ctl.exe tools/pd-ctl/main.go
+echo "bin/pd-ctl.exe"
+go build -o bin/pd-tso-bench.exe tools/pd-tso-bench/main.go
+echo "bin/pd-tso-bench.exe"
+go build -o bin/pd-recover.exe tools/pd-recover/main.go
+echo "bin/pd-recover.exe"

client/client.go

@@ -52,6 +52,11 @@ type Client interface {
 	GetPrevRegion(ctx context.Context, key []byte) (*metapb.Region, *metapb.Peer, error)
 	// GetRegionByID gets a region and its leader Peer from PD by id.
 	GetRegionByID(ctx context.Context, regionID uint64) (*metapb.Region, *metapb.Peer, error)
+	// ScanRegion gets a list of regions, starts from the region that contains key.
+	// Limit limits the maximum number of regions returned.
+	// If a region has no leader, corresponding leader will be placed by a peer
+	// with empty value (PeerID is 0).
+	ScanRegions(ctx context.Context, key []byte, limit int) ([]*metapb.Region, []*metapb.Peer, error)
 	// GetStore gets a store from PD by store id.
 	// The store may expire later. Caller is responsible for caching and taking care
 	// of store change.
@@ -681,6 +686,28 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*metapb.Re
 	return resp.GetRegion(), resp.GetLeader(), nil
 }
 
+func (c *client) ScanRegions(ctx context.Context, key []byte, limit int) ([]*metapb.Region, []*metapb.Peer, error) {
+	if span := opentracing.SpanFromContext(ctx); span != nil {
+		span = opentracing.StartSpan("pdclient.ScanRegions", opentracing.ChildOf(span.Context()))
+		defer span.Finish()
+	}
+	start := time.Now()
+	defer cmdDuration.WithLabelValues("scan_regions").Observe(time.Since(start).Seconds())
+	ctx, cancel := context.WithTimeout(ctx, pdTimeout)
+	resp, err := c.leaderClient().ScanRegions(ctx, &pdpb.ScanRegionsRequest{
+		Header:   c.requestHeader(),
+		StartKey: key,
+		Limit:    int32(limit),
+	})
+	cancel()
+	if err != nil {
+		cmdFailedDuration.WithLabelValues("scan_regions").Observe(time.Since(start).Seconds())
+		c.ScheduleCheckLeader()
+		return nil, nil, errors.WithStack(err)
+	}
+	return resp.GetRegions(), resp.GetLeaders(), nil
+}
+
 func (c *client) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) {
 	if span := opentracing.SpanFromContext(ctx); span != nil {
 		span = opentracing.StartSpan("pdclient.GetStore", opentracing.ChildOf(span.Context()))

client/client_test.go

@@ -260,6 +260,64 @@ func (s *testClientSuite) TestGetPrevRegion(c *C) {
 	c.Succeed()
 }
 
+func (s *testClientSuite) TestScanRegions(c *C) {
+	regionLen := 10
+	regions := make([]*metapb.Region, 0, regionLen)
+	for i := 0; i < regionLen; i++ {
+		regionID := regionIDAllocator.alloc()
+		r := &metapb.Region{
+			Id: regionID,
+			RegionEpoch: &metapb.RegionEpoch{
+				ConfVer: 1,
+				Version: 1,
+			},
+			StartKey: []byte{byte(i)},
+			EndKey:   []byte{byte(i + 1)},
+			Peers:    peers,
+		}
+		regions = append(regions, r)
+		req := &pdpb.RegionHeartbeatRequest{
+			Header: newHeader(s.srv),
+			Region: r,
+			Leader: peers[0],
+		}
+		err := s.regionHeartbeat.Send(req)
+		c.Assert(err, IsNil)
+	}
+
+	// Wait for region heartbeats.
+	testutil.WaitUntil(c, func(c *C) bool {
+		scanRegions, _, err := s.client.ScanRegions(context.Background(), []byte{0}, 10)
+		return err == nil && len(scanRegions) == 10
+	})
+
+	// Set leader of region3 to nil.
+	region3 := core.NewRegionInfo(regions[3], nil)
+	s.srv.GetRaftCluster().HandleRegionHeartbeat(region3)
+
+	check := func(start []byte, limit int, expect []*metapb.Region) {
+		scanRegions, leaders, err := s.client.ScanRegions(context.Background(), start, limit)
+		c.Assert(err, IsNil)
+		c.Assert(scanRegions, HasLen, len(expect))
+		c.Assert(leaders, HasLen, len(expect))
+		c.Log("scanRegions", scanRegions)
+		c.Log("expect", expect)
+		c.Log("scanLeaders", leaders)
+		for i := range expect {
+			c.Assert(scanRegions[i], DeepEquals, expect[i])
+			if scanRegions[i].GetId() == region3.GetID() {
+				c.Assert(leaders[i], DeepEquals, &metapb.Peer{})
+			} else {
+				c.Assert(leaders[i], DeepEquals, expect[i].Peers[0])
+			}
+		}
+	}
+
+	check([]byte{0}, 10, regions)
+	check([]byte{1}, 5, regions[1:6])
+	check([]byte{100}, 1, nil)
+}
+
 func (s *testClientSuite) TestGetRegionByID(c *C) {
 	regionID := regionIDAllocator.alloc()
 	region := &metapb.Region{

conf/config.toml

@@ -64,10 +64,10 @@ max-snapshot-count = 3
 max-pending-peer-count = 16
 max-store-down-time = "30m"
 leader-schedule-limit = 4
-region-schedule-limit = 4
-replica-schedule-limit = 8
+region-schedule-limit = 1024
+replica-schedule-limit = 1024
 merge-schedule-limit = 8
-tolerant-size-ratio = 5.0
+tolerant-size-ratio = 0
 
 # customized schedulers, the format is as below
 # if empty, it will use balance-leader, balance-region, hot-region as default

go.mod

@@ -29,8 +29,8 @@ require (
 	github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
 	github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9
 	github.com/pingcap/errors v0.10.1 // indirect
-	github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3
-	github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d
+	github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c
+	github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c
 	github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.8.0