From f15ea88093211ecc77a67df17cbbb62d75a1a367 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 11 Jan 2024 18:53:53 +0800 Subject: [PATCH 1/5] planner: return the complete error info when Parallel Apply meets problem Signed-off-by: Weizhen Wang --- pkg/planner/core/optimizer.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/planner/core/optimizer.go b/pkg/planner/core/optimizer.go index 925671337c49c..6ee284cfb94aa 100644 --- a/pkg/planner/core/optimizer.go +++ b/pkg/planner/core/optimizer.go @@ -1116,9 +1116,13 @@ func enableParallelApply(sctx sessionctx.Context, plan PhysicalPlan) PhysicalPla if noOrder && supportClone { apply.Concurrency = sctx.GetSessionVars().ExecutorConcurrency } else { - sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackErrorf("Some apply operators can not be executed in parallel")) + if err != nil { + sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackErrorf("Some apply operators can not be executed in parallel: %v", err)) + } + if !noOrder { + sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("Some apply operators can not be executed in parallel because of no order")) + } } - // because of the limitation 3, we cannot parallelize Apply operators in this Apply's inner size, // so we only invoke recursively for its outer child. apply.SetChild(outerIdx, enableParallelApply(sctx, apply.Children()[outerIdx])) From e0f043c38e44c11e77d876218e7276f64422c11a Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 11 Jan 2024 18:58:54 +0800 Subject: [PATCH 2/5] * Signed-off-by: Weizhen Wang --- pkg/planner/core/optimizer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/planner/core/optimizer.go b/pkg/planner/core/optimizer.go index 6ee284cfb94aa..fcad670fe94b3 100644 --- a/pkg/planner/core/optimizer.go +++ b/pkg/planner/core/optimizer.go @@ -1116,11 +1116,12 @@ func enableParallelApply(sctx sessionctx.Context, plan PhysicalPlan) PhysicalPla if noOrder && supportClone { apply.Concurrency = sctx.GetSessionVars().ExecutorConcurrency } else { - if err != nil { + if err != nil && noOrder { sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackErrorf("Some apply operators can not be executed in parallel: %v", err)) - } - if !noOrder { + } else if err == nil && !noOrder { sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("Some apply operators can not be executed in parallel because of no order")) + } else if err != nil && !noOrder { + sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackErrorf("Some apply operators can not be executed in parallel : %v and no order", err)) } } // because of the limitation 3, we cannot parallelize Apply operators in this Apply's inner size, From 6c2331c2544a7a255e1cf04338598613dc79ea93 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 11 Jan 2024 19:01:15 +0800 Subject: [PATCH 3/5] * Signed-off-by: Weizhen Wang --- pkg/planner/core/optimizer.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/planner/core/optimizer.go b/pkg/planner/core/optimizer.go index fcad670fe94b3..552e1181b6844 100644 --- a/pkg/planner/core/optimizer.go +++ b/pkg/planner/core/optimizer.go @@ -1116,12 +1116,10 @@ func enableParallelApply(sctx sessionctx.Context, plan PhysicalPlan) PhysicalPla if noOrder && supportClone { apply.Concurrency = sctx.GetSessionVars().ExecutorConcurrency } else { - if err != nil && noOrder { + if err != nil { sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackErrorf("Some apply operators can not be executed in parallel: %v", err)) - } else if err == nil && !noOrder { - sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("Some apply operators can not be executed in parallel because of no order")) - } else if err != nil && !noOrder { - sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackErrorf("Some apply operators can not be executed in parallel : %v and no order", err)) + } else { + sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("Some apply operators can not be executed in parallel")) } } // because of the limitation 3, we cannot parallelize Apply operators in this Apply's inner size, From 2fda1c52b0e98ad6d25324fbc9be0b434b812bde Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 12 Jan 2024 11:46:27 +0800 Subject: [PATCH 4/5] * Signed-off-by: Weizhen Wang --- .../core/casetest/parallelapply/BUILD.bazel | 16 +++++++++ .../core/casetest/parallelapply/main_test.go | 36 +++++++++++++++++++ .../parallelapply/parallel_apply_test.go | 33 +++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 pkg/planner/core/casetest/parallelapply/BUILD.bazel create mode 100644 pkg/planner/core/casetest/parallelapply/main_test.go create mode 100644 pkg/planner/core/casetest/parallelapply/parallel_apply_test.go diff --git a/pkg/planner/core/casetest/parallelapply/BUILD.bazel b/pkg/planner/core/casetest/parallelapply/BUILD.bazel new file mode 100644 index 0000000000000..0d472ac151e57 --- /dev/null +++ b/pkg/planner/core/casetest/parallelapply/BUILD.bazel @@ -0,0 +1,16 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "parallelapply_test", + timeout = "short", + srcs = [ + "main_test.go", + "parallel_apply_test.go", + ], + flaky = True, + deps = [ + "//pkg/testkit", + "//pkg/testkit/testsetup", + "@org_uber_go_goleak//:goleak", + ], +) diff --git a/pkg/planner/core/casetest/parallelapply/main_test.go b/pkg/planner/core/casetest/parallelapply/main_test.go new file mode 100644 index 0000000000000..db264e894a228 --- /dev/null +++ b/pkg/planner/core/casetest/parallelapply/main_test.go @@ -0,0 +1,36 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parallelapply + +import ( + "testing" + + "github.com/pingcap/tidb/pkg/testkit/testsetup" + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + testsetup.SetupForCommonTest() + opts := []goleak.Option{ + goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"), + goleak.IgnoreTopFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), + goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"), + goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun"), + goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + } + goleak.VerifyTestMain(m, opts...) +} diff --git a/pkg/planner/core/casetest/parallelapply/parallel_apply_test.go b/pkg/planner/core/casetest/parallelapply/parallel_apply_test.go new file mode 100644 index 0000000000000..80016173d79cb --- /dev/null +++ b/pkg/planner/core/casetest/parallelapply/parallel_apply_test.go @@ -0,0 +1,33 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parallelapply + +import ( + "testing" + + "github.com/pingcap/tidb/pkg/testkit" +) + +func TestParallelApplyWarnning(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t1 (a int, b int, c int);") + tk.MustExec("create table t2 (a int, b int, c int, key(a));") + tk.MustExec("create table t3(a int, b int, c int, key(a));") + tk.MustExec("set tidb_enable_parallel_apply=on;") + tk.MustQuery("select (select 1 from t2, t3 where t2.a=t3.a and t2.b > t1.b) from t1;") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Some apply operators can not be executed in parallel: *core.PhysicalIndexHashJoin doesn't support cloning")) +} From 3548ad7a1ca682042b02d8b8a7fafb26c4a0e81a Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 12 Jan 2024 11:48:52 +0800 Subject: [PATCH 5/5] * Signed-off-by: Weizhen Wang --- pkg/planner/core/casetest/parallelapply/main_test.go | 2 +- pkg/planner/core/casetest/parallelapply/parallel_apply_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/planner/core/casetest/parallelapply/main_test.go b/pkg/planner/core/casetest/parallelapply/main_test.go index db264e894a228..84c0876d8a1fe 100644 --- a/pkg/planner/core/casetest/parallelapply/main_test.go +++ b/pkg/planner/core/casetest/parallelapply/main_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 PingCAP, Inc. +// Copyright 2024 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/planner/core/casetest/parallelapply/parallel_apply_test.go b/pkg/planner/core/casetest/parallelapply/parallel_apply_test.go index 80016173d79cb..5df87d06d42ef 100644 --- a/pkg/planner/core/casetest/parallelapply/parallel_apply_test.go +++ b/pkg/planner/core/casetest/parallelapply/parallel_apply_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 PingCAP, Inc. +// Copyright 2024 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License.