Merge branch 'release-3.1' into cherry-pick-298-to-3.1

pingcap · May 29, 2020 · e89eb5c · e89eb5c
2 parents 66dafdc + 5437159
commit e89eb5c
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 2 deletions.
diff --git a/pkg/restore/import.go b/pkg/restore/import.go
@@ -190,6 +190,10 @@ func (importer *FileImporter) Import(
 		endKey = file.EndKey
 	} else {
 		startKey, endKey, err = rewriteFileKeys(file, rewriteRules)
+		// if not truncateRowKey here, if will scan one more region
+		// TODO need more test to check here
+		// startKey = truncateRowKey(startKey)
+		// endKey = truncateRowKey(endKey)
 	}
 	if err != nil {
 		return err
@@ -248,6 +252,12 @@ func (importer *FileImporter) Import(
 					switch e {
 					case errRewriteRuleNotFound, errRangeIsEmpty:
 						// Skip this region
+						log.Warn("download file skipped",
+							zap.Stringer("file", file),
+							zap.Stringer("region", info.Region),
+							zap.Binary("startKey", startKey),
+							zap.Binary("endKey", endKey),
+							zap.Error(errDownload))
 						continue regionLoop
 					}
 				}
@@ -366,6 +376,7 @@ func (importer *FileImporter) downloadSST(
 	}
 	log.Debug("download SST",
 		zap.Stringer("sstMeta", &sstMeta),
+		zap.Stringer("file", file),
 		zap.Stringer("region", regionInfo.Region),
 	)
 	var resp *import_sstpb.DownloadResponse

diff --git a/pkg/restore/util.go b/pkg/restore/util.go
@@ -103,11 +103,29 @@ func getSSTMetaFromFile(
 	if bytes.Compare(rangeStart, region.GetStartKey()) < 0 {
 		rangeStart = region.GetStartKey()
 	}
-	rangeEnd := append(append([]byte{}, regionRule.GetNewKeyPrefix()...), 0xff)
+
+	// Append 10 * 0xff to make sure rangeEnd cover all file key
+	// If choose to regionRule.NewKeyPrefix + 1, it may cause WrongPrefix here
+	// https://github.com/tikv/tikv/blob/970a9bf2a9ea782a455ae579ad237aaf6cb1daec/
+	// components/sst_importer/src/sst_importer.rs#L221
+	suffix := []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+	rangeEnd := append(append([]byte{}, regionRule.GetNewKeyPrefix()...), suffix...)
 	// rangeEnd = min(rangeEnd, region.EndKey)
 	if len(region.GetEndKey()) > 0 && bytes.Compare(rangeEnd, region.GetEndKey()) > 0 {
 		rangeEnd = region.GetEndKey()
 	}
+
+	if bytes.Compare(rangeStart, rangeEnd) > 0 {
+		log.Fatal("range start exceed range end",
+			zap.Binary("start", rangeStart),
+			zap.Binary("end", rangeEnd))
+	}
+
+	log.Debug("get sstMeta",
+		zap.Stringer("file", file),
+		zap.Binary("rangeStart", rangeStart),
+		zap.Binary("rangeEnd", rangeEnd))
+
 	return import_sstpb.SSTMeta{
 		Uuid:   id,
 		CfName: cfName,

diff --git a/pkg/restore/util_test.go b/pkg/restore/util_test.go
@@ -35,7 +35,7 @@ func (s *testRestoreUtilSuite) TestGetSSTMetaFromFile(c *C) {
 	}
 	sstMeta := getSSTMetaFromFile([]byte{}, file, region, rule)
 	c.Assert(string(sstMeta.GetRange().GetStart()), Equals, "t2abc")
-	c.Assert(string(sstMeta.GetRange().GetEnd()), Equals, "t2\xff")
+	c.Assert(string(sstMeta.GetRange().GetEnd()), Equals, "t2\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff")
 }
 
 func (s *testRestoreUtilSuite) TestValidateFileRanges(c *C) {

diff --git a/tests/br_range/run.sh b/tests/br_range/run.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+#
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+DB="$TEST_NAME"
+
+run_sql "create schema $DB;"
+
+run_sql "create table $DB.sbtest(id bigint primary key, c char(120) not null);"
+run_sql "insert into $DB.sbtest values (9223372036854775807, 'test');"
+run_sql "insert into $DB.sbtest values (9187343239835811840, 'test');"
+
+run_sql "create table $DB.sbtest2(id bigint unsigned primary key, c char(120) not null);"
+run_sql "insert into $DB.sbtest2 values (18446744073709551615, 'test');"
+run_sql "insert into $DB.sbtest2 values (9223372036854775808, 'test');"
+
+# backup db
+echo "backup start..."
+run_br backup db --db "$DB" -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+
+run_sql "drop schema $DB;"
+
+# restore db
+echo "restore start..."
+run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+
+run_sql "drop schema $DB;"
diff --git a/tools/go.sum b/tools/go.sum
@@ -235,8 +235,10 @@ github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/ryancurrah/gomodguard v1.0.2/go.mod h1:9T/Cfuxs5StfsocWr4WzDL36HqnX0fVb9d5fSEaLhoE=
 github.com/ryancurrah/gomodguard v1.0.4 h1:oCreMAt9GuFXDe9jW4HBpc3GjdX3R/sUEcLAGh1zPx8=
 github.com/ryancurrah/gomodguard v1.0.4/go.mod h1:9T/Cfuxs5StfsocWr4WzDL36HqnX0fVb9d5fSEaLhoE=
+github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE=
 github.com/securego/gosec v0.0.0-20200316084457-7da9f46445fd h1:qB+l4fYZsH78xORC1aqVS0zNmgkQp4rkj2rvfxQMtzc=
 github.com/securego/gosec v0.0.0-20200316084457-7da9f46445fd/go.mod h1:NurAFZsWJAEZjogSwdVPlHkOZB3DOAU7gsPP8VFZCHc=
 github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc=