Skip to content

Commit

Permalink
Merge branch 'master' into spot-test-issue-20658
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhouqiang-cl
zhouqiang-cl authored May 20, 2021
2 parents 4984f43 + 6bfc5f8 commit d83eabb
Show file tree
Hide file tree
Showing 17 changed files with 423 additions and 90 deletions.
18 changes: 9 additions & 9 deletions ddl/db_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2614,24 +2614,24 @@ func (s *testIntegrationSuite5) TestDropColumnsWithMultiIndex(c *C) {
tk.MustQuery(query).Check(testkit.Rows())
}

func (s *testIntegrationSuite5) TestDropLastVisibleColumn(c *C) {
func (s *testSerialDBSuite) TestDropLastVisibleColumnOrColumns(c *C) {
defer config.RestoreFunc()
config.UpdateGlobal(func(conf *config.Config) {
conf.Experimental.AllowsExpressionIndex = true
})
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test_db")
tk.MustExec("create table t_drop_last_column(x int, key((1+1)))")
defer tk.MustExec("drop table if exists t_drop_last_column")
_, err := tk.Exec("alter table t_drop_last_column drop column x")
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[ddl:1113]A table must have at least 1 column")
}

func (s *testIntegrationSuite5) TestDropLastVisibleColumns(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test_db")
// for visible columns
tk.MustExec("create table t_drop_last_columns(x int, y int, key((1+1)))")
defer tk.MustExec("drop table if exists t_drop_last_columns")
_, err := tk.Exec("alter table t_drop_last_columns drop column x, drop column y")
_, err = tk.Exec("alter table t_drop_last_columns drop column x, drop column y")
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[ddl:1113]A table must have at least 1 column")

tk.MustExec("drop table if exists t_drop_last_column, t_drop_last_columns")
}

func (s *testIntegrationSuite7) TestAutoIncrementTableOption(c *C) {
Expand Down
9 changes: 5 additions & 4 deletions executor/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -706,10 +706,11 @@ func (b *executorBuilder) buildSimple(v *plannercore.Simple) Executor {
base := newBaseExecutor(b.ctx, v.Schema(), v.ID())
base.initCap = chunk.ZeroCapacity
e := &SimpleExec{
baseExecutor: base,
Statement: v.Statement,
IsFromRemote: v.IsFromRemote,
is: b.is,
baseExecutor: base,
Statement: v.Statement,
IsFromRemote: v.IsFromRemote,
is: b.is,
StalenessTxnOption: v.StalenessTxnOption,
}
return e
}
Expand Down
4 changes: 3 additions & 1 deletion executor/distsql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1212,7 +1212,9 @@ func (w *tableWorker) executeTask(ctx context.Context, task *lookupTableTask) er
obtainedHandlesMap.Set(handle, true)
}

logutil.Logger(ctx).Error("inconsistent index handles", zap.String("index", w.idxLookup.index.Name.O),
logutil.Logger(ctx).Error("inconsistent index handles",
zap.String("table_name", w.idxLookup.index.Table.O),
zap.String("index", w.idxLookup.index.Name.O),
zap.Int("index_cnt", handleCnt), zap.Int("table_cnt", len(task.rows)),
zap.String("missing_handles", fmt.Sprint(GetLackHandles(task.handles, obtainedHandlesMap))),
zap.String("total_handles", fmt.Sprint(task.handles)))
Expand Down
32 changes: 26 additions & 6 deletions executor/simple.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ type SimpleExec struct {
IsFromRemote bool
done bool
is infoschema.InfoSchema

// StalenessTxnOption is used to execute the staleness txn during a read-only begin statement.
StalenessTxnOption *sessionctx.StalenessTxnOption
}

func (e *baseExecutor) getSysSession() (sessionctx.Context, error) {
Expand Down Expand Up @@ -566,13 +569,16 @@ func (e *SimpleExec) executeUse(s *ast.UseStmt) error {
}

func (e *SimpleExec) executeBegin(ctx context.Context, s *ast.BeginStmt) error {
// If `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND` is the first statement in TxnCtx, we should
// If `START TRANSACTION READ ONLY` is the first statement in TxnCtx, we should
// always create a new Txn instead of reusing it.
if s.ReadOnly {
enableNoopFuncs := e.ctx.GetSessionVars().EnableNoopFuncs
if !enableNoopFuncs && s.Bound == nil {
if !enableNoopFuncs && s.AsOf == nil && s.Bound == nil {
return expression.ErrFunctionsNoopImpl.GenWithStackByArgs("READ ONLY")
}
if s.AsOf != nil {
return e.executeStartTransactionReadOnlyWithBoundedStaleness(ctx, s)
}
if s.Bound != nil {
return e.executeStartTransactionReadOnlyWithTimestampBound(ctx, s)
}
Expand Down Expand Up @@ -614,6 +620,22 @@ func (e *SimpleExec) executeBegin(ctx context.Context, s *ast.BeginStmt) error {
return nil
}

func (e *SimpleExec) executeStartTransactionReadOnlyWithBoundedStaleness(ctx context.Context, s *ast.BeginStmt) error {
if e.StalenessTxnOption == nil {
return errors.New("Failed to get timestamp during start transaction read only as of timestamp")
}
if err := e.ctx.NewTxnWithStalenessOption(ctx, *e.StalenessTxnOption); err != nil {
return err
}

// With START TRANSACTION, autocommit remains disabled until you end
// the transaction with COMMIT or ROLLBACK. The autocommit mode then
// reverts to its previous state.
e.ctx.GetSessionVars().SetInTxn(true)
return nil
}

// TODO: deprecate this syntax and only keep `AS OF TIMESTAMP` statement.
func (e *SimpleExec) executeStartTransactionReadOnlyWithTimestampBound(ctx context.Context, s *ast.BeginStmt) error {
opt := sessionctx.StalenessTxnOption{}
opt.Mode = s.Bound.Mode
Expand All @@ -632,8 +654,7 @@ func (e *SimpleExec) executeStartTransactionReadOnlyWithTimestampBound(ctx conte
if err != nil {
return err
}
startTS := oracle.GoTimeToTS(gt)
opt.StartTS = startTS
opt.StartTS = oracle.GoTimeToTS(gt)
case ast.TimestampBoundExactStaleness:
// TODO: support funcCallExpr in future
v, ok := s.Bound.Timestamp.(*driver.ValueExpr)
Expand Down Expand Up @@ -668,8 +689,7 @@ func (e *SimpleExec) executeStartTransactionReadOnlyWithTimestampBound(ctx conte
if err != nil {
return err
}
startTS := oracle.GoTimeToTS(gt)
opt.StartTS = startTS
opt.StartTS = oracle.GoTimeToTS(gt)
}
err := e.ctx.NewTxnWithStalenessOption(ctx, opt)
if err != nil {
Expand Down
118 changes: 89 additions & 29 deletions executor/stale_txn_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,30 @@ func (s *testStaleTxnSerialSuite) TestExactStalenessTransaction(c *C) {
zone: "sz",
},
{
name: "begin",
name: "begin after TimestampBoundReadTimestamp",
preSQL: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`,
sql: "begin",
IsStaleness: false,
txnScope: kv.GlobalTxnScope,
zone: "",
},
{
name: "AsOfTimestamp",
preSQL: "begin",
sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00';`,
IsStaleness: true,
expectPhysicalTS: 1599321600000,
txnScope: "local",
zone: "sh",
},
{
name: "begin after AsOfTimestamp",
preSQL: `START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00';`,
sql: "begin",
IsStaleness: false,
txnScope: oracle.GlobalTxnScope,
zone: "",
},
}
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
Expand All @@ -106,8 +123,8 @@ func (s *testStaleTxnSerialSuite) TestExactStalenessTransaction(c *C) {
}
c.Assert(tk.Se.GetSessionVars().TxnCtx.IsStaleness, Equals, testcase.IsStaleness)
tk.MustExec("commit")
failpoint.Disable("github.com/pingcap/tidb/config/injectTxnScope")
}
failpoint.Disable("github.com/pingcap/tidb/config/injectTxnScope")
}

func (s *testStaleTxnSerialSuite) TestStaleReadKVRequest(c *C) {
Expand Down Expand Up @@ -147,13 +164,17 @@ func (s *testStaleTxnSerialSuite) TestStaleReadKVRequest(c *C) {
failpoint.Enable("github.com/pingcap/tidb/config/injectTxnScope", fmt.Sprintf(`return("%v")`, testcase.zone))
failpoint.Enable("github.com/pingcap/tidb/store/tikv/assertStoreLabels", fmt.Sprintf(`return("%v_%v")`, placement.DCLabelKey, testcase.txnScope))
failpoint.Enable("github.com/pingcap/tidb/store/tikv/assertStaleReadFlag", `return(true)`)
// Using NOW() will cause the loss of fsp precision, so we use NOW(3) to be accurate to the millisecond.
tk.MustExec(`START TRANSACTION READ ONLY AS OF TIMESTAMP NOW(3);`)
tk.MustQuery(testcase.sql)
tk.MustExec(`commit`)
tk.MustExec(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND EXACT STALENESS '00:00:00';`)
tk.MustQuery(testcase.sql)
tk.MustExec(`commit`)
failpoint.Disable("github.com/pingcap/tidb/config/injectTxnScope")
failpoint.Disable("github.com/pingcap/tidb/store/tikv/assertStoreLabels")
failpoint.Disable("github.com/pingcap/tidb/store/tikv/assertStaleReadFlag")
}
failpoint.Disable("github.com/pingcap/tidb/config/injectTxnScope")
failpoint.Disable("github.com/pingcap/tidb/store/tikv/assertStoreLabels")
failpoint.Disable("github.com/pingcap/tidb/store/tikv/assertStaleReadFlag")
}

func (s *testStaleTxnSerialSuite) TestStalenessAndHistoryRead(c *C) {
Expand All @@ -169,6 +190,17 @@ func (s *testStaleTxnSerialSuite) TestStalenessAndHistoryRead(c *C) {
tk.MustExec(updateSafePoint)
// set @@tidb_snapshot before staleness txn
tk.MustExec(`set @@tidb_snapshot="2016-10-08 16:45:26";`)
tk.MustExec(`START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00';`)
// 1599321600000 == 2020-09-06 00:00:00
c.Assert(oracle.ExtractPhysical(tk.Se.GetSessionVars().TxnCtx.StartTS), Equals, int64(1599321600000))
tk.MustExec("commit")
// set @@tidb_snapshot during staleness txn
tk.MustExec(`START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00';`)
tk.MustExec(`set @@tidb_snapshot="2016-10-08 16:45:26";`)
c.Assert(oracle.ExtractPhysical(tk.Se.GetSessionVars().TxnCtx.StartTS), Equals, int64(1599321600000))
tk.MustExec("commit")
// set @@tidb_snapshot before staleness txn
tk.MustExec(`set @@tidb_snapshot="2016-10-08 16:45:26";`)
tk.MustExec(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`)
c.Assert(oracle.ExtractPhysical(tk.Se.GetSessionVars().TxnCtx.StartTS), Equals, int64(1599321600000))
tk.MustExec("commit")
Expand All @@ -190,60 +222,76 @@ func (s *testStaleTxnSerialSuite) TestTimeBoundedStalenessTxn(c *C) {
name string
sql string
injectSafeTS uint64
useSafeTS bool
// compareWithSafeTS will be 0 if StartTS==SafeTS, -1 if StartTS < SafeTS, and +1 if StartTS > SafeTS.
compareWithSafeTS int
}{
{
name: "max 20 seconds ago, safeTS 10 secs ago",
sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MAX STALENESS '00:00:20'`,
injectSafeTS: func() uint64 {
return oracle.GoTimeToTS(time.Now().Add(-10 * time.Second))
}(),
useSafeTS: true,
name: "max 20 seconds ago, safeTS 10 secs ago",
sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MAX STALENESS '00:00:20'`,
injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-10 * time.Second)),
compareWithSafeTS: 0,
},
{
name: "max 10 seconds ago, safeTS 20 secs ago",
sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MAX STALENESS '00:00:10'`,
injectSafeTS: func() uint64 {
return oracle.GoTimeToTS(time.Now().Add(-20 * time.Second))
}(),
useSafeTS: false,
name: "max 10 seconds ago, safeTS 20 secs ago",
sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MAX STALENESS '00:00:10'`,
injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-20 * time.Second)),
compareWithSafeTS: 1,
},
{
name: "max 20 seconds ago, safeTS 10 secs ago",
sql: func() string {
return fmt.Sprintf(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MIN READ TIMESTAMP '%v'`,
time.Now().Add(-20*time.Second).Format("2006-01-02 15:04:05"))
}(),
injectSafeTS: func() uint64 {
return oracle.GoTimeToTS(time.Now().Add(-10 * time.Second))
}(),
useSafeTS: true,
injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-10 * time.Second)),
compareWithSafeTS: 0,
},
{
name: "max 10 seconds ago, safeTS 20 secs ago",
sql: func() string {
return fmt.Sprintf(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MIN READ TIMESTAMP '%v'`,
time.Now().Add(-10*time.Second).Format("2006-01-02 15:04:05"))
}(),
injectSafeTS: func() uint64 {
return oracle.GoTimeToTS(time.Now().Add(-20 * time.Second))
}(),
useSafeTS: false,
injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-20 * time.Second)),
compareWithSafeTS: 1,
},
{
name: "20 seconds ago to now, safeTS 10 secs ago",
sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP tidb_bounded_staleness(NOW() - INTERVAL 20 SECOND, NOW())`,
injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-10 * time.Second)),
compareWithSafeTS: 0,
},
{
name: "10 seconds ago to now, safeTS 20 secs ago",
sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP tidb_bounded_staleness(NOW() - INTERVAL 10 SECOND, NOW())`,
injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-20 * time.Second)),
compareWithSafeTS: 1,
},
{
name: "20 seconds ago to 10 seconds ago, safeTS 5 secs ago",
sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP tidb_bounded_staleness(NOW() - INTERVAL 20 SECOND, NOW() - INTERVAL 10 SECOND)`,
injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-5 * time.Second)),
compareWithSafeTS: -1,
},
}
for _, testcase := range testcases {
c.Log(testcase.name)
c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/injectSafeTS",
fmt.Sprintf("return(%v)", testcase.injectSafeTS)), IsNil)
c.Assert(failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS",
fmt.Sprintf("return(%v)", testcase.injectSafeTS)), IsNil)
tk.MustExec(testcase.sql)
if testcase.useSafeTS {
if testcase.compareWithSafeTS == 1 {
c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Greater, testcase.injectSafeTS)
} else if testcase.compareWithSafeTS == 0 {
c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Equals, testcase.injectSafeTS)
} else {
c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Greater, testcase.injectSafeTS)
c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Less, testcase.injectSafeTS)
}
tk.MustExec("commit")
failpoint.Disable("github.com/pingcap/tidb/store/tikv/injectSafeTS")
}
failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS")
failpoint.Disable("github.com/pingcap/tidb/store/tikv/injectSafeTS")
}

func (s *testStaleTxnSerialSuite) TestStalenessTransactionSchemaVer(c *C) {
Expand All @@ -263,4 +311,16 @@ func (s *testStaleTxnSerialSuite) TestStalenessTransactionSchemaVer(c *C) {
schemaVer3 := tk.Se.GetSessionVars().GetInfoSchema().SchemaMetaVersion()
// got an old infoSchema
c.Assert(schemaVer3, Equals, schemaVer1)

schemaVer4 := tk.Se.GetSessionVars().GetInfoSchema().SchemaMetaVersion()
time.Sleep(time.Second)
tk.MustExec("create table t (id int primary key);")
schemaVer5 := tk.Se.GetSessionVars().GetInfoSchema().SchemaMetaVersion()
// confirm schema changed
c.Assert(schemaVer4, Less, schemaVer5)

tk.MustExec(`START TRANSACTION READ ONLY AS OF TIMESTAMP NOW() - INTERVAL 1 SECOND`)
schemaVer6 := tk.Se.GetSessionVars().GetInfoSchema().SchemaMetaVersion()
// got an old infoSchema
c.Assert(schemaVer6, Equals, schemaVer4)
}
44 changes: 41 additions & 3 deletions expression/builtin_json.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
package expression

import (
json2 "encoding/json"
"bytes"
goJSON "encoding/json"
"strconv"
"strings"

Expand Down Expand Up @@ -841,7 +842,7 @@ func (b *builtinJSONValidStringSig) evalInt(row chunk.Row) (res int64, isNull bo
}

data := hack.Slice(val)
if json2.Valid(data) {
if goJSON.Valid(data) {
res = 1
} else {
res = 0
Expand Down Expand Up @@ -1072,8 +1073,45 @@ type jsonPrettyFunctionClass struct {
baseFunctionClass
}

type builtinJSONSPrettySig struct {
baseBuiltinFunc
}

func (b *builtinJSONSPrettySig) Clone() builtinFunc {
newSig := &builtinJSONSPrettySig{}
newSig.cloneFrom(&b.baseBuiltinFunc)
return newSig
}

func (c *jsonPrettyFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) {
return nil, errFunctionNotExists.GenWithStackByArgs("FUNCTION", "JSON_PRETTY")
if err := c.verifyArgs(args); err != nil {
return nil, err
}

bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETString, types.ETJson)
if err != nil {
return nil, err
}
sig := &builtinJSONSPrettySig{bf}
sig.setPbCode(tipb.ScalarFuncSig_JsonPrettySig)
return sig, nil
}

func (b *builtinJSONSPrettySig) evalString(row chunk.Row) (res string, isNull bool, err error) {
obj, isNull, err := b.args[0].EvalJSON(b.ctx, row)
if isNull || err != nil {
return res, isNull, err
}

buf, err := obj.MarshalJSON()
if err != nil {
return res, isNull, err
}
var resBuf bytes.Buffer
if err = goJSON.Indent(&resBuf, buf, "", " "); err != nil {
return res, isNull, err
}
return resBuf.String(), false, nil
}

type jsonQuoteFunctionClass struct {
Expand Down
Loading

0 comments on commit d83eabb

Please sign in to comment.