Cherry pick bug fixes to release-2.0 (#1324)

* server, client: fix hanging problem when etcd failed to start (#1267)

* server: use same initialcluster config to restart joined member (#1279)

* fix server build

* pdctl: cherry pick bugfixes (#1298, #1299, #1308)

* server/api: fix the issue about `regions/check` API (#1311)

* fix join build

* fix pdctl build

* fix region test

* fix warnings

.travis.yml

@@ -1,14 +1,14 @@
 language: go
 
 go:
-  - 1.9.x
+  - 1.11.x
 
 script:
   - make dev
 
 matrix:
   include:
-  - go: 1.9.x
+  - go: 1.11.x
     install:
       - go get github.com/mattn/goveralls
     env:

circle.yml

@@ -3,7 +3,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: golang:1.8.1
+      - image: golang:1.11.1
     working_directory: /go/src/github.com/pingcap/pd
     steps:
       - checkout

pd-client/client.go

@@ -153,7 +153,9 @@ func (c *client) initClusterID() error {
 	defer cancel()
 	for i := 0; i < maxInitClusterRetries; i++ {
 		for _, u := range c.urls {
-			members, err := c.getMembers(ctx, u)
+			timeoutCtx, timeoutCancel := context.WithTimeout(ctx, pdTimeout)
+			members, err := c.getMembers(timeoutCtx, u)
+			timeoutCancel()
 			if err != nil || members.GetHeader() == nil {
 				log.Errorf("[pd] failed to get cluster id: %v", err)
 				continue

pdctl/command/region_command.go

@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"strconv"
+	"strings"
 
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/spf13/cobra"
@@ -122,7 +124,7 @@ func showRegionTopReadCommandFunc(cmd *cobra.Command, args []string) {
 // NewRegionWithKeyCommand return a region with key subcommand of regionCmd
 func NewRegionWithKeyCommand() *cobra.Command {
 	r := &cobra.Command{
-		Use:   "key [--format=raw|pb|proto|protobuf] <key>",
+		Use:   "key [--format=raw|encode] <key>",
 		Short: "show the region with key",
 		Run:   showRegionWithTableCommandFunc,
 	}
@@ -145,8 +147,8 @@ func showRegionWithTableCommandFunc(cmd *cobra.Command, args []string) {
 	switch format {
 	case "raw":
 		key = args[0]
-	case "pb", "proto", "protobuf":
-		key, err = decodeProtobufText(args[0])
+	case "encode":
+		key, err = decodeKey(args[0])
 		if err != nil {
 			fmt.Println("Error: ", err)
 			return
@@ -155,18 +157,18 @@ func showRegionWithTableCommandFunc(cmd *cobra.Command, args []string) {
 		fmt.Println("Error: unknown format")
 		return
 	}
-	// TODO: Deal with path escaped
+
+	key = url.QueryEscape(key)
 	prefix := regionKeyPrefix + "/" + key
 	r, err := doRequest(cmd, prefix, http.MethodGet)
 	if err != nil {
 		fmt.Printf("Failed to get region: %s\n", err)
 		return
 	}
 	fmt.Println(r)
-
 }
 
-func decodeProtobufText(text string) (string, error) {
+func decodeKey(text string) (string, error) {
 	var buf []byte
 	r := bytes.NewBuffer([]byte(text))
 	for {
@@ -177,13 +179,32 @@ func decodeProtobufText(text string) (string, error) {
 			}
 			break
 		}
-		if c == '\\' {
-			_, err := fmt.Sscanf(string(r.Next(3)), "%03o", &c)
+		if c != '\\' {
+			buf = append(buf, c)
+			continue
+		}
+		n := r.Next(1)
+		if len(n) == 0 {
+			return "", io.EOF
+		}
+		// See: https://golang.org/ref/spec#Rune_literals
+		if idx := strings.IndexByte(`abfnrtv\'"`, n[0]); idx != -1 {
+			buf = append(buf, []byte("\a\b\f\n\r\t\v\\'\"")[idx])
+			continue
+		}
+
+		switch n[0] {
+		case 'x':
+			fmt.Sscanf(string(r.Next(2)), "%02x", &c)
+			buf = append(buf, c)
+		default:
+			n = append(n, r.Next(2)...)
+			_, err := fmt.Sscanf(string(n), "%03o", &c)
 			if err != nil {
 				return "", err
 			}
+			buf = append(buf, c)
 		}
-		buf = append(buf, c)
 	}
 	return string(buf), nil
 }

pkg/faketikv/client.go

@@ -68,12 +68,12 @@ func NewClient(pdAddr string, tag string) (Client, <-chan *pdpb.RegionHeartbeatR
 	simutil.Logger.Infof("[%s][pd] create pd client with endpoints %v", tag, pdAddr)
 	ctx, cancel := context.WithCancel(context.Background())
 	c := &client{
-		url: pdAddr,
+		url:                      pdAddr,
 		reportRegionHeartbeatCh:  make(chan *core.RegionInfo, 1),
 		receiveRegionHeartbeatCh: make(chan *pdpb.RegionHeartbeatResponse, 1),
-		ctx:    ctx,
-		cancel: cancel,
-		tag:    tag,
+		ctx:                      ctx,
+		cancel:                   cancel,
+		tag:                      tag,
 	}
 	cc, err := c.createConn()
 	if err != nil {

pkg/faketikv/node.go

@@ -78,13 +78,13 @@ func NewNode(id uint64, addr string, pdAddr string) (*Node, error) {
 		return nil, err
 	}
 	return &Node{
-		Store:  store,
-		stats:  stats,
-		client: client,
-		ctx:    ctx,
-		cancel: cancel,
-		tasks:  make(map[uint64]Task),
-		state:  Down,
+		Store:                   store,
+		stats:                   stats,
+		client:                  client,
+		ctx:                     ctx,
+		cancel:                  cancel,
+		tasks:                   make(map[uint64]Task),
+		state:                   Down,
 		reciveRegionHeartbeatCh: reciveRegionHeartbeatCh,
 	}, nil
 }

server/api/region.go

@@ -122,73 +122,82 @@ func newRegionsHandler(svr *server.Server, rd *render.Render) *regionsHandler {
 	}
 }
 
+func convertToAPIRegions(regions []*core.RegionInfo) *regionsInfo {
+	regionInfos := make([]*regionInfo, len(regions))
+	for i, r := range regions {
+		regionInfos[i] = newRegionInfo(r)
+	}
+	return &regionsInfo{
+		Count:   len(regions),
+		Regions: regionInfos,
+	}
+}
+
 func (h *regionsHandler) GetAll(w http.ResponseWriter, r *http.Request) {
 	cluster := h.svr.GetRaftCluster()
 	if cluster == nil {
 		h.rd.JSON(w, http.StatusInternalServerError, server.ErrNotBootstrapped.Error())
 		return
 	}
 
-	regions := cluster.GetMetaRegions()
-	regionInfos := make([]*regionInfo, len(regions))
-	for i, r := range regions {
-		regionInfos[i] = newRegionInfo(&core.RegionInfo{Region: r})
-	}
-	regionsInfo := &regionsInfo{
-		Count:   len(regions),
-		Regions: regionInfos,
-	}
+	regions := cluster.GetRegions()
+	regionsInfo := convertToAPIRegions(regions)
 	h.rd.JSON(w, http.StatusOK, regionsInfo)
 }
 
 func (h *regionsHandler) GetMissPeerRegions(w http.ResponseWriter, r *http.Request) {
 	handler := h.svr.GetHandler()
-	res, err := handler.GetMissPeerRegions()
+	regions, err := handler.GetMissPeerRegions()
 	if err != nil {
 		h.rd.JSON(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	h.rd.JSON(w, http.StatusOK, res)
+	regionsInfo := convertToAPIRegions(regions)
+	h.rd.JSON(w, http.StatusOK, regionsInfo)
 }
 
 func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, r *http.Request) {
 	handler := h.svr.GetHandler()
-	res, err := handler.GetExtraPeerRegions()
+	regions, err := handler.GetExtraPeerRegions()
 	if err != nil {
 		h.rd.JSON(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	h.rd.JSON(w, http.StatusOK, res)
+	regionsInfo := convertToAPIRegions(regions)
+	h.rd.JSON(w, http.StatusOK, regionsInfo)
 }
 
 func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, r *http.Request) {
 	handler := h.svr.GetHandler()
-	res, err := handler.GetPendingPeerRegions()
+	regions, err := handler.GetPendingPeerRegions()
 	if err != nil {
 		h.rd.JSON(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	h.rd.JSON(w, http.StatusOK, res)
+	regionsInfo := convertToAPIRegions(regions)
+	h.rd.JSON(w, http.StatusOK, regionsInfo)
 }
 
 func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, r *http.Request) {
 	handler := h.svr.GetHandler()
-	res, err := handler.GetDownPeerRegions()
+	regions, err := handler.GetDownPeerRegions()
 	if err != nil {
 		h.rd.JSON(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	h.rd.JSON(w, http.StatusOK, res)
+	regionsInfo := convertToAPIRegions(regions)
+	h.rd.JSON(w, http.StatusOK, regionsInfo)
 }
 
 func (h *regionsHandler) GetIncorrectNamespaceRegions(w http.ResponseWriter, r *http.Request) {
 	handler := h.svr.GetHandler()
-	res, err := handler.GetIncorrectNamespaceRegions()
+	regions, err := handler.GetIncorrectNamespaceRegions()
 	if err != nil {
 		h.rd.JSON(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	h.rd.JSON(w, http.StatusOK, res)
+	regionsInfo := convertToAPIRegions(regions)
+	h.rd.JSON(w, http.StatusOK, regionsInfo)
 }
 
 func (h *regionsHandler) GetRegionSiblings(w http.ResponseWriter, r *http.Request) {
@@ -247,15 +256,8 @@ func (h *regionsHandler) GetTopNRegions(w http.ResponseWriter, r *http.Request,
 		limit = maxRegionLimit
 	}
 	regions := topNRegions(cluster.GetRegions(), less, limit)
-	regionInfos := make([]*regionInfo, len(regions))
-	for i, r := range regions {
-		regionInfos[i] = newRegionInfo(r)
-	}
-	res := &regionsInfo{
-		Count:   len(regions),
-		Regions: regionInfos,
-	}
-	h.rd.JSON(w, http.StatusOK, res)
+	regionsInfo := convertToAPIRegions(regions)
+	h.rd.JSON(w, http.StatusOK, regionsInfo)
 }
 
 // RegionHeap implements heap.Interface, used for selecting top n regions.

server/api/region_test.go

@@ -16,9 +16,12 @@ package api
 import (
 	"fmt"
 	"math/rand"
+	"net/url"
+	"sort"
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/kvproto/pkg/metapb"
+	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/pd/server"
 	"github.com/pingcap/pd/server/core"
 )
@@ -79,6 +82,58 @@ func (s *testRegionSuite) TestRegion(c *C) {
 	c.Assert(r2, DeepEquals, newRegionInfo(r))
 }
 
+func (s *testRegionSuite) TestRegionCheck(c *C) {
+	r := newTestRegionInfo(2, 1, []byte("a"), []byte("b"))
+	downPeer := &metapb.Peer{Id: 13, StoreId: 2}
+	r = r.Clone()
+	r.Peers = append(r.Peers, downPeer)
+	r.DownPeers = []*pdpb.PeerStats{{Peer: downPeer, DownSeconds: 3600}}
+	r.PendingPeers = []*metapb.Peer{downPeer}
+	mustRegionHeartbeat(c, s.svr, r)
+	url := fmt.Sprintf("%s/region/id/%d", s.urlPrefix, r.GetId())
+	r1 := &regionInfo{}
+	err := readJSONWithURL(url, r1)
+	c.Assert(err, IsNil)
+	c.Assert(r1, DeepEquals, newRegionInfo(r))
+
+	url = fmt.Sprintf("%s/regions/check/%s", s.urlPrefix, "down-peer")
+	r2 := &regionsInfo{}
+	err = readJSONWithURL(url, r2)
+	c.Assert(err, IsNil)
+	c.Assert(r2, DeepEquals, &regionsInfo{Count: 1, Regions: []*regionInfo{newRegionInfo(r)}})
+
+	url = fmt.Sprintf("%s/regions/check/%s", s.urlPrefix, "pending-peer")
+	r3 := &regionsInfo{}
+	err = readJSONWithURL(url, r3)
+	c.Assert(err, IsNil)
+	c.Assert(r3, DeepEquals, &regionsInfo{Count: 1, Regions: []*regionInfo{newRegionInfo(r)}})
+}
+
+func (s *testRegionSuite) TestRegions(c *C) {
+	rs := []*core.RegionInfo{
+		newTestRegionInfo(2, 1, []byte("a"), []byte("b")),
+		newTestRegionInfo(3, 1, []byte("b"), []byte("c")),
+		newTestRegionInfo(4, 2, []byte("c"), []byte("d")),
+	}
+	regions := make([]*regionInfo, 0, len(rs))
+	for _, r := range rs {
+		regions = append(regions, newRegionInfo(r))
+		mustRegionHeartbeat(c, s.svr, r)
+	}
+	url := fmt.Sprintf("%s/regions", s.urlPrefix)
+	regionsInfo := &regionsInfo{}
+	err := readJSONWithURL(url, regionsInfo)
+	c.Assert(err, IsNil)
+	c.Assert(regionsInfo.Count, Equals, len(regions))
+	sort.Slice(regionsInfo.Regions, func(i, j int) bool {
+		return regionsInfo.Regions[i].ID < regionsInfo.Regions[j].ID
+	})
+	for i, r := range regionsInfo.Regions {
+		c.Assert(r.ID, Equals, regions[i].ID)
+		c.Assert(r.ApproximateSize, Equals, regions[i].ApproximateSize)
+	}
+}
+
 func (s *testRegionSuite) TestTopFlow(c *C) {
 	r1 := newTestRegionInfo(1, 1, []byte("a"), []byte("b"))
 	r1.WrittenBytes, r1.ReadBytes = 1000, 1000
@@ -125,3 +180,38 @@ func (s *testRegionSuite) TestTopN(c *C) {
 		}
 	}
 }
+
+var _ = Suite(&testGetRegionSuite{})
+
+type testGetRegionSuite struct {
+	svr       *server.Server
+	cleanup   cleanUpFunc
+	urlPrefix string
+}
+
+func (s *testGetRegionSuite) SetUpSuite(c *C) {
+	s.svr, s.cleanup = mustNewServer(c)
+	mustWaitLeader(c, []*server.Server{s.svr})
+
+	addr := s.svr.GetAddr()
+	s.urlPrefix = fmt.Sprintf("%s%s/api/v1", addr, apiPrefix)
+
+	mustBootstrapCluster(c, s.svr)
+}
+
+func (s *testGetRegionSuite) TearDownSuite(c *C) {
+	s.cleanup()
+}
+
+func (s *testGetRegionSuite) TestRegionKey(c *C) {
+	r := newTestRegionInfo(99, 1, []byte{0xFF, 0xFF, 0xAA}, []byte{0xFF, 0xFF, 0xCC})
+	r.WrittenBytes = 500
+	r.ReadBytes = 800
+	r.RegionEpoch = &metapb.RegionEpoch{Version: 2, ConfVer: 3}
+	mustRegionHeartbeat(c, s.svr, r)
+	url := fmt.Sprintf("%s/region/key/%s", s.urlPrefix, url.QueryEscape(string([]byte{0xFF, 0xFF, 0xBB})))
+	regionInfo := &regionInfo{}
+	err := readJSONWithURL(url, regionInfo)
+	c.Assert(err, IsNil)
+	c.Assert(r.GetId(), Equals, regionInfo.ID)
+}