Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

meta: make auto increment id can be adjust. (#10978) #11006

Merged
merged 3 commits into from
Jul 1, 2019
Merged

meta: make auto increment id can be adjust. (#10978) #11006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4fb9097
meta: make auto increment id can be adjust. (#10978)
Jun 28, 2019
53711d9
fix ci
Jul 1, 2019
8ac1b92
Merge branch 'release-3.0' into autoAdjust
jackysp Jul 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions ddl/db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

. "github.com/pingcap/check"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
tmysql "github.com/pingcap/parser/mysql"
Expand Down Expand Up @@ -2163,7 +2164,11 @@ func (s *testDBSuite4) TestComment(c *C) {
s.tk.MustExec("drop table if exists ct, ct1")
}

func (s *testDBSuite5) TestRebaseAutoID(c *C) {
func (s *testDBSuite4) TestRebaseAutoID(c *C) {
c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
defer func() {
c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
}()
s.tk = testkit.NewTestKit(c, s.store)
s.tk.MustExec("use " + s.schemaName)

Expand Down Expand Up @@ -2758,7 +2763,12 @@ func (s *testDBSuite1) TestModifyColumnCharset(c *C) {

}

func (s *testDBSuite2) TestAlterShardRowIDBits(c *C) {
func (s *testDBSuite4) TestAlterShardRowIDBits(c *C) {
c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
defer func() {
c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
}()

s.tk = testkit.NewTestKit(c, s.store)
tk := s.tk

Expand Down
4 changes: 4 additions & 0 deletions ddl/serial_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,10 @@ func (s *testSerialSuite) TestRecoverTableByJobID(c *C) {
}

func (s *testSerialSuite) TestRecoverTableByTableName(c *C) {
c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
defer func() {
c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
}()
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("create database if not exists test_recover")
tk.MustExec("use test_recover")
Expand Down
5 changes: 5 additions & 0 deletions executor/ddl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"time"

. "github.com/pingcap/check"
"github.com/pingcap/failpoint"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/parser/terror"
Expand Down Expand Up @@ -343,6 +344,10 @@ func (s *testSuite3) TestDefaultDBAfterDropCurDB(c *C) {
}

func (s *testSuite3) TestRenameTable(c *C) {
c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
defer func() {
c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
}()
tk := testkit.NewTestKit(c, s.store)

tk.MustExec("create database rename1")
Expand Down
4 changes: 4 additions & 0 deletions executor/seqtest/seq_executor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -708,6 +708,10 @@ func checkGoroutineExists(keyword string) bool {
}

func (s *seqTestSuite) TestAdminShowNextID(c *C) {
c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
defer func() {
c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
}()
step := int64(10)
autoIDStep := autoid.GetStep()
autoid.SetStep(step)
Expand Down
53 changes: 44 additions & 9 deletions meta/autoid/autoid.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (

"github.com/cznic/mathutil"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/parser/terror"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
Expand All @@ -30,6 +31,12 @@ import (
"go.uber.org/zap"
)

const (
minStep = 1000
maxStep = 2000000
defaultConsumeTime = 10 * time.Second
)

// Test needs to change it, so it's a variable.
var step = int64(30000)

Expand Down Expand Up @@ -59,8 +66,10 @@ type allocator struct {
end int64
store kv.Storage
// dbID is current database's ID.
dbID int64
isUnsigned bool
dbID int64
isUnsigned bool
lastAllocTime time.Time
step int64
}

// GetStep is only used by tests
Expand Down Expand Up @@ -124,7 +133,7 @@ func (alloc *allocator) rebase4Unsigned(tableID int64, requiredBase uint64, allo
uCurrentEnd := uint64(currentEnd)
if allocIDs {
newBase = mathutil.MaxUint64(uCurrentEnd, requiredBase)
newEnd = mathutil.MinUint64(math.MaxUint64-uint64(step), newBase) + uint64(step)
newEnd = mathutil.MinUint64(math.MaxUint64-uint64(alloc.step), newBase) + uint64(alloc.step)
} else {
if uCurrentEnd >= requiredBase {
newBase = uCurrentEnd
Expand Down Expand Up @@ -169,7 +178,7 @@ func (alloc *allocator) rebase4Signed(tableID, requiredBase int64, allocIDs bool
}
if allocIDs {
newBase = mathutil.MaxInt64(currentEnd, requiredBase)
newEnd = mathutil.MinInt64(math.MaxInt64-step, newBase) + step
newEnd = mathutil.MinInt64(math.MaxInt64-alloc.step, newBase) + alloc.step
} else {
if currentEnd >= requiredBase {
newBase = currentEnd
Expand Down Expand Up @@ -215,21 +224,24 @@ func (alloc *allocator) alloc4Unsigned(tableID int64) (int64, error) {
if alloc.base == alloc.end { // step
var newBase, newEnd int64
startTime := time.Now()
consumeDur := startTime.Sub(alloc.lastAllocTime)
alloc.step = NextStep(alloc.step, consumeDur)
err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error {
m := meta.NewMeta(txn)
var err1 error
newBase, err1 = m.GetAutoTableID(alloc.dbID, tableID)
if err1 != nil {
return err1
}
tmpStep := int64(mathutil.MinUint64(math.MaxUint64-uint64(newBase), uint64(step)))
tmpStep := int64(mathutil.MinUint64(math.MaxUint64-uint64(newBase), uint64(alloc.step)))
newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep)
return err1
})
metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
if err != nil {
return 0, err
}
alloc.lastAllocTime = time.Now()
if uint64(newBase) == math.MaxUint64 {
return 0, ErrAutoincReadFailed
}
Expand All @@ -248,21 +260,24 @@ func (alloc *allocator) alloc4Signed(tableID int64) (int64, error) {
if alloc.base == alloc.end { // step
var newBase, newEnd int64
startTime := time.Now()
consumeDur := startTime.Sub(alloc.lastAllocTime)
alloc.step = NextStep(alloc.step, consumeDur)
err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error {
m := meta.NewMeta(txn)
var err1 error
newBase, err1 = m.GetAutoTableID(alloc.dbID, tableID)
if err1 != nil {
return err1
}
tmpStep := mathutil.MinInt64(math.MaxInt64-newBase, step)
tmpStep := mathutil.MinInt64(math.MaxInt64-newBase, alloc.step)
newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep)
return err1
})
metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
if err != nil {
return 0, err
}
alloc.lastAllocTime = time.Now()
if newBase == math.MaxInt64 {
return 0, ErrAutoincReadFailed
}
Expand Down Expand Up @@ -290,12 +305,32 @@ func (alloc *allocator) Alloc(tableID int64) (int64, error) {
return alloc.alloc4Signed(tableID)
}

// NextStep return new auto id step according to previous step and consuming time.
func NextStep(curStep int64, consumeDur time.Duration) int64 {
failpoint.Inject("mockAutoIDChange", func(val failpoint.Value) {
if val.(bool) {
failpoint.Return(step)
}
})

consumeRate := defaultConsumeTime.Seconds() / consumeDur.Seconds()
res := int64(float64(curStep) * consumeRate)
if res < minStep {
return minStep
} else if res > maxStep {
return maxStep
}
return res
}

// NewAllocator returns a new auto increment id generator on the store.
func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool) Allocator {
return &allocator{
store: store,
dbID: dbID,
isUnsigned: isUnsigned,
store: store,
dbID: dbID,
isUnsigned: isUnsigned,
step: step,
lastAllocTime: time.Now(),
}
}

Expand Down
21 changes: 21 additions & 0 deletions meta/autoid/autoid_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

. "github.com/pingcap/check"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/parser/model"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
Expand All @@ -39,6 +40,11 @@ type testSuite struct {
}

func (*testSuite) TestT(c *C) {
c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
defer func() {
c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
}()

store, err := mockstore.NewMockTikvStore()
c.Assert(err, IsNil)
defer store.Close()
Expand Down Expand Up @@ -130,6 +136,11 @@ func (*testSuite) TestT(c *C) {
}

func (*testSuite) TestUnsignedAutoid(c *C) {
c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
defer func() {
c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
}()

store, err := mockstore.NewMockTikvStore()
c.Assert(err, IsNil)
defer store.Close()
Expand Down Expand Up @@ -315,3 +326,13 @@ func (*testSuite) TestRollbackAlloc(c *C) {
c.Assert(alloc.Base(), Equals, int64(0))
c.Assert(alloc.End(), Equals, int64(0))
}

// TestNextStep tests generate next auto id step.
func (*testSuite) TestNextStep(c *C) {
nextStep := autoid.NextStep(2000000, 1*time.Nanosecond)
c.Assert(nextStep, Equals, int64(2000000))
nextStep = autoid.NextStep(678910, 10*time.Second)
c.Assert(nextStep, Equals, int64(678910))
nextStep = autoid.NextStep(50000, 10*time.Minute)
c.Assert(nextStep, Equals, int64(1000))
}