Merge branch 'master' of https://github.com/pingcap/tidb into pingcap…

…#18056

Makefile

@@ -44,7 +44,7 @@ ARCH      := "`uname -s`"
 LINUX     := "Linux"
 MAC       := "Darwin"
 PACKAGE_LIST  := go list ./...| grep -vE "cmd"
-PACKAGES  := $$($(PACKAGE_LIST))
+PACKAGES  ?= $$($(PACKAGE_LIST))
 PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)/||'
 FILES     := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go")
 
@@ -160,7 +160,7 @@ test_part_1: checklist explaintest
 
 test_part_2: checkdep gotest gogenerate
 
-explaintest: server
+explaintest: server_check
 	@cd cmd/explaintest && ./run-tests.sh -s ../../bin/tidb-server
 
 ddltest:

cmd/explaintest/r/generated_columns.result

@@ -1,3 +1,4 @@
+set @@tidb_partition_prune_mode='dynamic-only';
 DROP TABLE IF EXISTS person;
 CREATE TABLE person (
 id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,

cmd/explaintest/r/select.result

@@ -376,7 +376,7 @@ explain select a != any (select a from t t2) from t t1;
 id	estRows	task	access object	operator info
 Projection_8	10000.00	root		and(or(or(gt(Column#8, 1), ne(test.t.a, Column#7)), if(ne(Column#9, 0), <nil>, 0)), and(ne(Column#10, 0), if(isnull(test.t.a), <nil>, 1)))->Column#11
 └─HashJoin_9	10000.00	root		CARTESIAN inner join
-  ├─StreamAgg_14(Build)	1.00	root		funcs:firstrow(Column#13)->Column#7, funcs:count(distinct Column#14)->Column#8, funcs:sum(Column#15)->Column#9, funcs:count(1)->Column#10
+  ├─StreamAgg_14(Build)	1.00	root		funcs:max(Column#13)->Column#7, funcs:count(distinct Column#14)->Column#8, funcs:sum(Column#15)->Column#9, funcs:count(1)->Column#10
   │ └─Projection_19	10000.00	root		test.t.a, test.t.a, cast(isnull(test.t.a), decimal(65,0) BINARY)->Column#15
   │   └─TableReader_18	10000.00	root		data:TableFullScan_17
   │     └─TableFullScan_17	10000.00	cop[tikv]	table:t2	keep order:false, stats:pseudo

cmd/explaintest/t/generated_columns.test

@@ -2,6 +2,7 @@
 -- Most of the cases are ported from other tests to make sure generated columns behaves the same.
 
 -- Stored generated columns as indices
+set @@tidb_partition_prune_mode='dynamic-only';
 
 DROP TABLE IF EXISTS person;
 CREATE TABLE person (

config/config.go

@@ -61,6 +61,8 @@ const (
 	DefStatusHost = "0.0.0.0"
 	// DefStoreLivenessTimeout is the default value for store liveness timeout.
 	DefStoreLivenessTimeout = "5s"
+	// DefTiDBRedactLog is the default value for redact log.
+	DefTiDBRedactLog = 0
 )
 
 // Valid config maps
@@ -158,6 +160,8 @@ type Config struct {
 	EnableGlobalIndex bool `toml:"enable-global-index" json:"enable-global-index"`
 	// DeprecateIntegerDisplayWidth indicates whether deprecating the max display length for integer.
 	DeprecateIntegerDisplayWidth bool `toml:"deprecate-integer-display-length" json:"deprecate-integer-display-length"`
+	// EnableRedactLog indicates that whether redact log, 0 is disable. 1 is enable.
+	EnableRedactLog int32 `toml:"enable-redact-log" json:"enable-redact-log"`
 }
 
 // UpdateTempStoragePath is to update the `TempStoragePath` if port/statusPort was changed
@@ -418,6 +422,7 @@ type Performance struct {
 	CommitterConcurrency int     `toml:"committer-concurrency" json:"committer-concurrency"`
 	MaxTxnTTL            uint64  `toml:"max-txn-ttl" json:"max-txn-ttl"`
 	MemProfileInterval   string  `toml:"mem-profile-interval" json:"mem-profile-interval"`
+	IndexUsageSyncLease  string  `toml:"index-usage-sync-lease" json:"index-usage-sync-lease"`
 }
 
 // PlanCache is the PlanCache section of the config.
@@ -550,8 +555,6 @@ type Plugin struct {
 
 // PessimisticTxn is the config for pessimistic transaction.
 type PessimisticTxn struct {
-	// Enable must be true for 'begin lock' or session variable to start a pessimistic transaction.
-	Enable bool `toml:"enable" json:"enable"`
 	// The max count of retry for a single statement in a pessimistic transaction.
 	MaxRetryCount uint `toml:"max-retry-count" json:"max-retry-count"`
 }
@@ -659,8 +662,9 @@ var defaultConf = Config{
 		TxnTotalSizeLimit:    DefTxnTotalSizeLimit,
 		DistinctAggPushDown:  false,
 		CommitterConcurrency: 16,
-		MaxTxnTTL:            10 * 60 * 1000, // 10min
+		MaxTxnTTL:            60 * 60 * 1000, // 1hour
 		MemProfileInterval:   "1m",
+		IndexUsageSyncLease:  "60s",
 	},
 	ProxyProtocol: ProxyProtocol{
 		Networks:      "",
@@ -711,7 +715,6 @@ var defaultConf = Config{
 		Strategy:     "range",
 	},
 	PessimisticTxn: PessimisticTxn{
-		Enable:        true,
 		MaxRetryCount: 256,
 	},
 	StmtSummary: StmtSummary{
@@ -736,6 +739,7 @@ var defaultConf = Config{
 		SpilledFileEncryptionMethod: SpilledFileEncryptionMethodPlaintext,
 	},
 	DeprecateIntegerDisplayWidth: false,
+	EnableRedactLog:              DefTiDBRedactLog,
 }
 
 var (
@@ -762,6 +766,7 @@ func StoreGlobalConfig(config *Config) {
 
 var deprecatedConfig = map[string]struct{}{
 	"pessimistic-txn.ttl":            {},
+	"pessimistic-txn.enable":         {},
 	"log.file.log-rotate":            {},
 	"log.log-slow-query":             {},
 	"txn-local-latches":              {},
@@ -978,6 +983,23 @@ var TableLockDelayClean = func() uint64 {
 	return GetGlobalConfig().DelayCleanTableLock
 }
 
+// RedactLogEnabled uses to check whether enabled the log redact.
+func RedactLogEnabled() bool {
+	return atomic.LoadInt32(&GetGlobalConfig().EnableRedactLog) == 1
+}
+
+// SetRedactLog uses to set log redact status.
+func SetRedactLog(enable bool) {
+	value := int32(0)
+	if enable {
+		value = 1
+	}
+	g := GetGlobalConfig()
+	newConf := *g
+	newConf.EnableRedactLog = value
+	StoreGlobalConfig(&newConf)
+}
+
 // ToLogConfig converts *Log to *logutil.LogConfig.
 func (l *Log) ToLogConfig() *logutil.LogConfig {
 	return logutil.NewLogConfig(l.Level, l.Format, l.SlowQueryFile, l.File, l.getDisableTimestamp(), func(config *zaplog.Config) { config.DisableErrorVerbose = l.getDisableErrorStack() })

config/config.toml.example

@@ -272,7 +272,7 @@ txn-entry-size-limit = 6291456
 committer-concurrency = 16
 
 # max lifetime of transaction ttl manager.
-max-txn-ttl = 600000
+max-txn-ttl = 3600000
 
 # the interval duration between two memory profile into global tracker
 mem-profile-interval = "1m"
@@ -420,9 +420,6 @@ binlog-socket = ""
 strategy = "range"
 
 [pessimistic-txn]
-# enable pessimistic transaction.
-enable = true
-
 # max retry count for a statement in a pessimistic transaction.
 max-retry-count = 256