Merge branch 'master' of github.com:pingcap/tidb into ld-worker

.github/dependabot.yml

@@ -11,4 +11,7 @@ updates:
       - dependency-name: "golang.org/*"
       - dependency-name: "github.com/golangci/golangci-lint"
     open-pull-requests-limit: 2
-
+  - package-ecosystem: "gomod"
+    directory: "/parser"
+    schedule:
+      interval: "weekly"

DEPS.bzl

@@ -210,8 +210,8 @@ def go_deps():
         name = "com_github_ashanbrown_forbidigo",
         build_file_proto_mode = "disable",
         importpath = "github.com/ashanbrown/forbidigo",
-        sum = "h1:VkYIwb/xxdireGAdJNZoo24O4lmnEWkactplBlWTShc=",
-        version = "v1.3.0",
+        sum = "h1:spdPbupaSqtWORq1Q4eHBoPBmHtwVyLKwaedbSLc5Sw=",
+        version = "v1.4.0",
     )
     go_repository(
         name = "com_github_ashanbrown_makezero",
@@ -364,8 +364,8 @@ def go_deps():
         name = "com_github_bombsimon_wsl_v3",
         build_file_proto_mode = "disable",
         importpath = "github.com/bombsimon/wsl/v3",
-        sum = "h1:Mka/+kRLoQJq7g2rggtgQsjuI/K5Efd87WX96EWFxjM=",
-        version = "v3.3.0",
+        sum = "h1:RkSxjT3tmlptwfgEgTgU+KYKLI35p/tviNXNXiL2aNU=",
+        version = "v3.4.0",
     )
     go_repository(
         name = "com_github_breeswish_gin_jwt_v2",
@@ -778,8 +778,8 @@ def go_deps():
         name = "com_github_daixiang0_gci",
         build_file_proto_mode = "disable",
         importpath = "github.com/daixiang0/gci",
-        sum = "h1:t8XZ0vK6l0pwPoOmoGyqW2NwQlvbpAQNVvu/GRBgykM=",
-        version = "v0.9.0",
+        sum = "h1:jBrwBmBZTDsGsXiaCTLIe9diotp1X4X64zodFrh7l+c=",
+        version = "v0.9.1",
     )
 
     go_repository(
@@ -1213,8 +1213,8 @@ def go_deps():
         name = "com_github_go_critic_go_critic",
         build_file_proto_mode = "disable",
         importpath = "github.com/go-critic/go-critic",
-        sum = "h1:fDaR/5GWURljXwF8Eh31T2GZNz9X4jeboS912mWF8Uo=",
-        version = "v0.6.5",
+        sum = "h1:1evPrElnLQ2LZtJfmNDzlieDhjnq36SLgNzisx06oPM=",
+        version = "v0.6.7",
     )
     go_repository(
         name = "com_github_go_echarts_go_echarts",
@@ -1378,8 +1378,8 @@ def go_deps():
         name = "com_github_go_toolsmith_astcast",
         build_file_proto_mode = "disable",
         importpath = "github.com/go-toolsmith/astcast",
-        sum = "h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=",
-        version = "v1.0.0",
+        sum = "h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8=",
+        version = "v1.1.0",
     )
     go_repository(
         name = "com_github_go_toolsmith_astcopy",
@@ -1392,36 +1392,36 @@ def go_deps():
         name = "com_github_go_toolsmith_astequal",
         build_file_proto_mode = "disable",
         importpath = "github.com/go-toolsmith/astequal",
-        sum = "h1:+LVdyRatFS+XO78SGV4I3TCEA0AC7fKEGma+fH+674o=",
-        version = "v1.0.3",
+        sum = "h1:kHKm1AWqClYn15R0K1KKE4RG614D46n+nqUQ06E1dTw=",
+        version = "v1.1.0",
     )
     go_repository(
         name = "com_github_go_toolsmith_astfmt",
         build_file_proto_mode = "disable",
         importpath = "github.com/go-toolsmith/astfmt",
-        sum = "h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=",
-        version = "v1.0.0",
+        sum = "h1:iJVPDPp6/7AaeLJEruMsBUlOYCmvg0MoCfJprsOmcco=",
+        version = "v1.1.0",
     )
     go_repository(
         name = "com_github_go_toolsmith_astp",
         build_file_proto_mode = "disable",
         importpath = "github.com/go-toolsmith/astp",
-        sum = "h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=",
-        version = "v1.0.0",
+        sum = "h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA=",
+        version = "v1.1.0",
     )
     go_repository(
         name = "com_github_go_toolsmith_strparse",
         build_file_proto_mode = "disable",
         importpath = "github.com/go-toolsmith/strparse",
-        sum = "h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=",
-        version = "v1.0.0",
+        sum = "h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw=",
+        version = "v1.1.0",
     )
     go_repository(
         name = "com_github_go_toolsmith_typep",
         build_file_proto_mode = "disable",
         importpath = "github.com/go-toolsmith/typep",
-        sum = "h1:8xdsa1+FSIH/RhEkgnD1j2CJOy5mNllW1Q9tRiYwvlk=",
-        version = "v1.0.2",
+        sum = "h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus=",
+        version = "v1.1.0",
     )
     go_repository(
         name = "com_github_go_xmlfmt_xmlfmt",
@@ -1606,8 +1606,8 @@ def go_deps():
         name = "com_github_golangci_golangci_lint",
         build_file_proto_mode = "disable",
         importpath = "github.com/golangci/golangci-lint",
-        sum = "h1:N5HD/x0ZrhJYsgKWyz7yJxxQ8JKR0Acc+FOP7QtGSAA=",
-        version = "v1.51.1",
+        sum = "h1:yIcsT1X9ZYHdSpeWXRT1ORC/FPGSqDHbHsu9uk4FK7M=",
+        version = "v1.51.2",
     )
     go_repository(
         name = "com_github_golangci_gosec",
@@ -2440,8 +2440,8 @@ def go_deps():
         name = "com_github_junk1tm_musttag",
         build_file_proto_mode = "disable",
         importpath = "github.com/junk1tm/musttag",
-        sum = "h1:VK4L7v7lvWAhKDDx0cUJgbb0UBNipYinv8pPeHJzH9Q=",
-        version = "v0.4.4",
+        sum = "h1:d+mpJ1vn6WFEVKHwkgJiIedis1u/EawKOuUTygAUtCo=",
+        version = "v0.4.5",
     )
 
     go_repository(
@@ -3445,8 +3445,8 @@ def go_deps():
         name = "com_github_polyfloyd_go_errorlint",
         build_file_proto_mode = "disable",
         importpath = "github.com/polyfloyd/go-errorlint",
-        sum = "h1:ZevdyEGxDoHAMQUVvdTT06hnYuKULe8TQkOmIYx6s1c=",
-        version = "v1.0.6",
+        sum = "h1:VKoEFg5yxSgJ2yFPVhxW7oGz+f8/OVcuMeNvcPIi6Eg=",
+        version = "v1.1.0",
     )
 
     go_repository(
@@ -3525,8 +3525,8 @@ def go_deps():
         name = "com_github_quasilyte_go_ruleguard",
         build_file_proto_mode = "disable",
         importpath = "github.com/quasilyte/go-ruleguard",
-        sum = "h1:sd+abO1PEI9fkYennwzHn9kl3nqP6M5vE7FiOzZ+5CE=",
-        version = "v0.3.18",
+        sum = "h1:tfMnabXle/HzOb5Xe9CUZYWXKfkS1KwRmZyPmD9nVcc=",
+        version = "v0.3.19",
     )
     go_repository(
         name = "com_github_quasilyte_go_ruleguard_dsl",
@@ -3539,8 +3539,8 @@ def go_deps():
         name = "com_github_quasilyte_gogrep",
         build_file_proto_mode = "disable",
         importpath = "github.com/quasilyte/gogrep",
-        sum = "h1:6Gtn2i04RD0gVyYf2/IUMTIs+qYleBt4zxDqkLTcu4U=",
-        version = "v0.0.0-20220828223005-86e4605de09f",
+        sum = "h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo=",
+        version = "v0.5.0",
     )
     go_repository(
         name = "com_github_quasilyte_regex_syntax",
@@ -3693,8 +3693,8 @@ def go_deps():
         name = "com_github_sashamelentyev_usestdlibvars",
         build_file_proto_mode = "disable",
         importpath = "github.com/sashamelentyev/usestdlibvars",
-        sum = "h1:GQGlReyL9Ek8DdJmwtwhHbhwHnuPfsKaprpjnrPcjxc=",
-        version = "v1.21.1",
+        sum = "h1:01h+/2Kd+NblNItNeux0veSL5cBF1jbEOPrEhDzGYq0=",
+        version = "v1.23.0",
     )
 
     go_repository(
@@ -3723,8 +3723,8 @@ def go_deps():
         name = "com_github_securego_gosec_v2",
         build_file_proto_mode = "disable",
         importpath = "github.com/securego/gosec/v2",
-        sum = "h1:U1hfs0oBackChXA72plCYVA4cOlQ4gO+209dHiSNZbI=",
-        version = "v2.14.0",
+        sum = "h1:v4Ym7FF58/jlykYmmhZ7mTm7FQvN/setNm++0fgIAtw=",
+        version = "v2.15.0",
     )
 
     go_repository(
@@ -4101,8 +4101,8 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sum = "h1:FYyqGriZ0yeCB6Zt2pZytGZuPbr9mzNOYzHHJtDRuq4=",
-        version = "v2.0.6-0.20230220051003-e0e6019f8ac7",
+        sum = "h1:hZ0aZfGAaS3rM0hYiMq6j0xHpVFgoYw6KeqrniGb7Zo=",
+        version = "v2.0.6-0.20230227032358-40a82457ebaa",
     )
     go_repository(
         name = "com_github_tikv_pd",
@@ -5888,8 +5888,8 @@ def go_deps():
         name = "org_golang_x_exp_typeparams",
         build_file_proto_mode = "disable",
         importpath = "golang.org/x/exp/typeparams",
-        sum = "h1:Jw5wfR+h9mnIYH+OtGT2im5wV1YGGDora5vTv/aa5bE=",
-        version = "v0.0.0-20221208152030-732eee02a75a",
+        sum = "h1:6WHiuFL9FNjg8RljAaT7FNUuKDbvMqS1i5cr2OE2sLQ=",
+        version = "v0.0.0-20230203172020-98cc5a0785f9",
     )
 
     go_repository(

br/pkg/checksum/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "//br/pkg/metautil",
         "//br/pkg/storage",
         "//br/pkg/summary",
+        "//br/pkg/utils",
         "//distsql",
         "//kv",
         "//parser/model",
@@ -21,6 +22,7 @@ go_library(
         "//util/ranger",
         "@com_github_gogo_protobuf//proto",
         "@com_github_pingcap_errors//:errors",
+        "@com_github_pingcap_failpoint//:failpoint",
         "@com_github_pingcap_kvproto//pkg/brpb",
         "@com_github_pingcap_log//:log",
         "@com_github_pingcap_tipb//go-tipb",
@@ -46,6 +48,7 @@ go_test(
         "//sessionctx/variable",
         "//testkit",
         "//testkit/testsetup",
+        "@com_github_pingcap_failpoint//:failpoint",
         "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",
     ],

br/pkg/checksum/executor.go

@@ -7,8 +7,10 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/log"
 	"github.com/pingcap/tidb/br/pkg/metautil"
+	"github.com/pingcap/tidb/br/pkg/utils"
 	"github.com/pingcap/tidb/distsql"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/parser/model"
@@ -330,13 +332,30 @@ func (exec *Executor) Execute(
 	updateFn func(),
 ) (*tipb.ChecksumResponse, error) {
 	checksumResp := &tipb.ChecksumResponse{}
+	checksumBackoffer := utils.InitialRetryState(utils.ChecksumRetryTime, utils.ChecksumWaitInterval, utils.ChecksumMaxWaitInterval)
 	for _, req := range exec.reqs {
 		// Pointer to SessionVars.Killed
 		// Killed is a flag to indicate that this query is killed.
 		//
 		// It is useful in TiDB, however, it's a place holder in BR.
 		killed := uint32(0)
-		resp, err := sendChecksumRequest(ctx, client, req, kv.NewVariables(&killed))
+		var (
+			resp *tipb.ChecksumResponse
+			err  error
+		)
+		err = utils.WithRetry(ctx, func() error {
+			resp, err = sendChecksumRequest(ctx, client, req, kv.NewVariables(&killed))
+			failpoint.Inject("checksumRetryErr", func(val failpoint.Value) {
+				// first time reach here. return error
+				if val.(bool) {
+					err = errors.New("inject checksum error")
+				}
+			})
+			if err != nil {
+				return errors.Trace(err)
+			}
+			return nil
+		}, &checksumBackoffer)
 		if err != nil {
 			return nil, errors.Trace(err)
 		}

br/pkg/checksum/executor_test.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"testing"
 
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/br/pkg/backup"
 	"github.com/pingcap/tidb/br/pkg/checksum"
 	"github.com/pingcap/tidb/br/pkg/metautil"
@@ -108,4 +109,12 @@ func TestChecksum(t *testing.T) {
 		}
 		return nil
 	}))
+
+	exe4, err := checksum.NewExecutorBuilder(tableInfo3, math.MaxUint64).Build()
+	require.NoError(t, err)
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/checksum/checksumRetryErr", `1*return(true)`))
+	failpoint.Disable("github.com/pingcap/tidb/br/pkg/checksum/checksumRetryErr")
+	resp4, err := exe4.Execute(context.TODO(), mock.Storage.GetClient(), func() {})
+	require.NoError(t, err)
+	require.NotNil(t, resp4)
 }

br/pkg/lightning/backend/local/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "local_unix_generic.go",
         "local_windows.go",
         "localhelper.go",
+        "region_job.go",
     ],
     importpath = "github.com/pingcap/tidb/br/pkg/lightning/backend/local",
     visibility = ["//visibility:public"],
@@ -28,7 +29,6 @@ go_library(
         "//br/pkg/lightning/manual",
         "//br/pkg/lightning/metric",
         "//br/pkg/lightning/tikv",
-        "//br/pkg/lightning/worker",
         "//br/pkg/logutil",
         "//br/pkg/membuf",
         "//br/pkg/pdutil",
@@ -40,6 +40,7 @@ go_library(
         "//kv",
         "//parser/model",
         "//parser/mysql",
+        "//store/pdtypes",
         "//table",
         "//tablecodec",
         "//types",
@@ -93,19 +94,18 @@ go_test(
         "local_check_test.go",
         "local_test.go",
         "localhelper_test.go",
+        "region_job_test.go",
     ],
     embed = [":local"],
     flaky = True,
     shard_count = 45,
     deps = [
-        "//br/pkg/errors",
         "//br/pkg/lightning/backend",
         "//br/pkg/lightning/backend/kv",
         "//br/pkg/lightning/common",
         "//br/pkg/lightning/glue",
         "//br/pkg/lightning/log",
         "//br/pkg/lightning/mydump",
-        "//br/pkg/lightning/worker",
         "//br/pkg/membuf",
         "//br/pkg/mock",
         "//br/pkg/pdutil",

br/pkg/lightning/backend/local/engine.go

@@ -988,7 +988,14 @@ func (e *Engine) newKVIter(ctx context.Context, opts *pebble.IterOptions) Iter {
 	return newDupDetectIter(ctx, e.db, e.keyAdapter, opts, e.duplicateDB, logger, e.dupDetectOpt)
 }
 
+// getFirstAndLastKey reads the first and last key in range [lowerBound, upperBound)
+// in the engine. Empty upperBound means unbounded.
 func (e *Engine) getFirstAndLastKey(lowerBound, upperBound []byte) ([]byte, []byte, error) {
+	if len(upperBound) == 0 {
+		// we use empty slice for unbounded upper bound, but it means max value in pebble
+		// so reset to nil
+		upperBound = nil
+	}
 	opt := &pebble.IterOptions{
 		LowerBound: lowerBound,
 		UpperBound: upperBound,

br/pkg/lightning/backend/local/engine_test.go

@@ -122,4 +122,9 @@ func TestGetFirstAndLastKey(t *testing.T) {
 	require.NoError(t, err)
 	require.Nil(t, first)
 	require.Nil(t, last)
+
+	first, last, err = f.getFirstAndLastKey([]byte("e"), []byte(""))
+	require.NoError(t, err)
+	require.Equal(t, []byte("e"), first)
+	require.Equal(t, []byte("e"), last)
 }