Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ddl: add recover for run ddl job #10981

Merged
merged 6 commits into from
Jul 2, 2019
+33 −2
Merged

ddl: add recover for run ddl job #10981

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ed7edb0
ddl: add recover for run ddl job
crazycs520 Jun 28, 2019
fe17a4a
address comment
crazycs520 Jun 28, 2019
912a642
address comment
crazycs520 Jun 28, 2019
8a2f415
address comment
crazycs520 Jul 1, 2019
407cb62
Merge branch 'master' into fix-ddl-panic
crazycs520 Jul 1, 2019
8bdc4d7
Merge branch 'master' into fix-ddl-panic
bb7133 Jul 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions ddl/ddl_worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"time"

"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/terror"
"github.com/pingcap/tidb/ddl/util"
Expand Down Expand Up @@ -471,6 +472,18 @@ func chooseLeaseTime(t, max time.Duration) time.Duration {

// runDDLJob runs a DDL job. It returns the current schema version in this transaction and the error.
func (w *worker) runDDLJob(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
defer func() {
r := recover()
zimulala marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we use WithRecovery？

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

if r != nil {
logutil.BgLogger().Error("run ddl job panic",
zap.Reflect("r", r),
zap.Stack("stack trace"))
job.State = model.JobStateCancelling
Copy link
Contributor

@winkyao winkyao Jun 28, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add some comments, why cancel the job here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

}
}()

failpoint.Inject("mockPanicInRunDDLJob", func(val failpoint.Value) {})

logutil.Logger(w.logCtx).Info("[ddl] run DDL job", zap.String("job", job.String()))
timeStart := time.Now()
defer func() {
Expand Down
14 changes: 14 additions & 0 deletions ddl/failtest/fail_db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,3 +386,17 @@ LOOP:
tk.MustExec("admin check table test_add_index")
tk.MustExec("drop table test_add_index")
}

// TestRunDDLJobPanic tests recover panic when run ddl job panic.
func (s *testFailDBSuite) TestRunDDLJobPanic(c *C) {
defer func() {
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/mockPanicInRunDDLJob"), IsNil)
}()
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/mockPanicInRunDDLJob", `1*panic("panic test")`), IsNil)
_, err := tk.Exec("create table t(c1 int, c2 int)")
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[ddl:12]cancelled DDL job")
}
2 changes: 1 addition & 1 deletion ddl/table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -773,7 +773,7 @@ func onModifyTableCharsetAndCollate(t *meta.Meta, job *model.Job) (ver int64, _

func checkTableNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, tableName string) error {
// d.infoHandle maybe nil in some test.
if d.infoHandle == nil {
if d.infoHandle == nil || d.infoHandle.IsValid() {
return checkTableNotExistsFromStore(t, schemaID, tableName)
}
// Try to use memory schema info to check first.
Expand Down
5 changes: 5 additions & 0 deletions infoschema/infoschema.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,11 @@ func (h *Handle) Get() InfoSchema {
return schema
}

// IsValid uses to check whether handle value is nil.
func (h *Handle) IsValid() bool {
winkyao marked this conversation as resolved.
Show resolved Hide resolved
return h.value.Load() == nil
}

// EmptyClone creates a new Handle with the same store and memSchema, but the value is not set.
func (h *Handle) EmptyClone() *Handle {
newHandle := &Handle{
Expand Down