Skip to content

Commit

Permalink
*: improve planner/core to make test faster (#41416)
Browse files Browse the repository at this point in the history
ref #41377
  • Loading branch information
hawkingrei authored Feb 15, 2023
1 parent aa3b57c commit 0e2ba9e
Show file tree
Hide file tree
Showing 79 changed files with 12,533 additions and 12,273 deletions.
13 changes: 2 additions & 11 deletions planner/core/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ go_library(
"//parser/opcode",
"//parser/terror",
"//parser/types",
"//planner/core/internal",
"//planner/funcdep",
"//planner/property",
"//planner/util",
Expand Down Expand Up @@ -169,14 +170,12 @@ go_test(
"expression_rewriter_test.go",
"expression_test.go",
"find_best_task_test.go",
"flat_plan_test.go",
"fragment_test.go",
"indexmerge_intersection_test.go",
"indexmerge_path_test.go",
"indexmerge_test.go",
"integration_partition_test.go",
"integration_test.go",
"logical_plan_test.go",
"logical_plan_trace_test.go",
"logical_plans_test.go",
"main_test.go",
Expand All @@ -200,17 +199,12 @@ go_test(
"plan_to_pb_test.go",
"planbuilder_test.go",
"point_get_plan_test.go",
"predicate_simplification_test.go",
"prepare_test.go",
"preprocess_test.go",
"rule_derive_topn_from_window_test.go",
"rule_inject_extra_projection_test.go",
"rule_join_reorder_dp_test.go",
"rule_join_reorder_test.go",
"rule_result_reorder_test.go",
"stats_test.go",
"stringer_test.go",
"window_push_down_test.go",
],
data = glob(["testdata/**"]),
embed = [":core"],
Expand All @@ -236,6 +230,7 @@ go_test(
"//parser/mysql",
"//parser/terror",
"//planner",
"//planner/core/internal",
"//planner/property",
"//planner/util",
"//session",
Expand All @@ -245,8 +240,6 @@ go_test(
"//sessiontxn",
"//statistics",
"//statistics/handle",
"//store/mockstore",
"//store/mockstore/unistore",
"//table",
"//testkit",
"//testkit/ddlhelper",
Expand Down Expand Up @@ -275,12 +268,10 @@ go_test(
"@com_github_golang_snappy//:snappy",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_pingcap_tipb//go-tipb",
"@com_github_prometheus_client_golang//prometheus",
"@com_github_prometheus_client_model//go",
"@com_github_stretchr_testify//require",
"@com_github_tikv_client_go_v2//testutils",
"@org_golang_x_exp//slices",
"@org_uber_go_goleak//:goleak",
],
Expand Down
113 changes: 0 additions & 113 deletions planner/core/binary_plan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,12 @@ import (
"fmt"
"io"
"os"
"regexp"
"strings"
"testing"

"github.com/golang/snappy"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/testkit/testdata"
Expand All @@ -36,117 +34,6 @@ import (
"github.com/stretchr/testify/require"
)

func simplifyAndCheckBinaryPlan(t *testing.T, pb *tipb.ExplainData) {
if pb.Main != nil {
simplifyAndCheckBinaryOperator(t, pb.Main, pb.WithRuntimeStats)
}
for _, cte := range pb.Ctes {
if cte != nil {
simplifyAndCheckBinaryOperator(t, cte, pb.WithRuntimeStats)
}
}
}

func simplifyAndCheckBinaryOperator(t *testing.T, pb *tipb.ExplainOperator, withRuntimeStats bool) {
if withRuntimeStats {
if pb.TaskType == tipb.TaskType_root {
require.NotEmpty(t, pb.RootBasicExecInfo)
} else if pb.TaskType != tipb.TaskType_unknown {
require.NotEmpty(t, pb.CopExecInfo)
}
}
pb.RootBasicExecInfo = ""
pb.RootGroupExecInfo = nil
pb.CopExecInfo = ""
match, err := regexp.MatchString("((Table|Index).*Scan)|CTEFullScan|Point_Get", pb.Name)
if err == nil && match {
require.NotNil(t, pb.AccessObjects)
}
// AccessObject field is an interface and json.Unmarshall can't handle it, so we don't check it against the json output.
pb.AccessObjects = nil
// MemoryBytes and DiskBytes are not stable sometimes.
pb.MemoryBytes = 0
pb.DiskBytes = 0
if len(pb.Children) > 0 {
for _, op := range pb.Children {
if op != nil {
simplifyAndCheckBinaryOperator(t, op, withRuntimeStats)
}
}
}
}

func TestBinaryPlanInExplainAndSlowLog(t *testing.T) {
// Prepare the slow log
originCfg := config.GetGlobalConfig()
newCfg := *originCfg
f, err := os.CreateTemp("", "tidb-slow-*.log")
require.NoError(t, err)
newCfg.Log.SlowQueryFile = f.Name()
config.StoreGlobalConfig(&newCfg)
defer func() {
config.StoreGlobalConfig(originCfg)
require.NoError(t, f.Close())
require.NoError(t, os.Remove(newCfg.Log.SlowQueryFile))
}()
require.NoError(t, logutil.InitLogger(newCfg.Log.ToLogConfig()))
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_cost_model_version=2")
// If we don't set this, it will be false sometimes and the cost in the result will be different.
tk.MustExec("set @@tidb_enable_chunk_rpc=true")
tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", f.Name()))
tk.MustExec("set tidb_slow_log_threshold=0;")
defer func() {
tk.MustExec("set tidb_slow_log_threshold=300;")
}()

var input []string
var output []struct {
SQL string
BinaryPlan *tipb.ExplainData
}
planSuiteData := core.GetBinaryPlanSuiteData()
planSuiteData.LoadTestCases(t, &input, &output)

for i, test := range input {
comment := fmt.Sprintf("case:%v sql:%s", i, test)
if len(test) < 7 || test[:7] != "explain" {
tk.MustExec(test)
testdata.OnRecord(func() {
output[i].SQL = test
output[i].BinaryPlan = nil
})
continue
}
result := testdata.ConvertRowsToStrings(tk.MustQuery(test).Rows())
require.Equal(t, len(result), 1, comment)
s := result[0]

// assert that the binary plan in the slow log is the same as the result in the EXPLAIN statement
slowLogResult := testdata.ConvertRowsToStrings(tk.MustQuery("select binary_plan from information_schema.slow_query " +
`where query = "` + test + `;" ` +
"order by time desc limit 1").Rows())
require.Lenf(t, slowLogResult, 1, comment)
require.Equal(t, s, slowLogResult[0], comment)

b, err := base64.StdEncoding.DecodeString(s)
require.NoError(t, err)
b, err = snappy.Decode(nil, b)
require.NoError(t, err)
binary := &tipb.ExplainData{}
err = binary.Unmarshal(b)
require.NoError(t, err)
testdata.OnRecord(func() {
output[i].SQL = test
output[i].BinaryPlan = binary
})
simplifyAndCheckBinaryPlan(t, binary)
require.Equal(t, output[i].BinaryPlan, binary)
}
}

func TestBinaryPlanSwitch(t *testing.T) {
originCfg := config.GetGlobalConfig()
newCfg := *originCfg
Expand Down
71 changes: 71 additions & 0 deletions planner/core/casetest/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
load("@io_bazel_rules_go//go:def.bzl", "go_test")

go_test(
name = "casetest_test",
timeout = "short",
srcs = [
"binary_plan_test.go",
"cbo_test.go",
"enforce_mpp_test.go",
"expression_rewriter_test.go",
"flat_plan_test.go",
"integration_partition_test.go",
"integration_test.go",
"main_test.go",
"partition_pruner_test.go",
"physical_plan_test.go",
"plan_test.go",
"point_get_plan_test.go",
"predicate_simplification_test.go",
"rule_derive_topn_from_window_test.go",
"rule_inject_extra_projection_test.go",
"rule_join_reorder_test.go",
"rule_result_reorder_test.go",
"stats_test.go",
"window_push_down_test.go",
],
data = glob(["testdata/**"]),
flaky = True,
shard_count = 50,
deps = [
"//config",
"//domain",
"//executor",
"//expression",
"//expression/aggregation",
"//infoschema",
"//kv",
"//parser",
"//parser/ast",
"//parser/model",
"//parser/mysql",
"//parser/terror",
"//planner",
"//planner/core",
"//planner/core/internal",
"//planner/property",
"//session",
"//sessionctx/stmtctx",
"//sessionctx/variable",
"//sessiontxn",
"//statistics",
"//statistics/handle",
"//testkit",
"//testkit/external",
"//testkit/testdata",
"//testkit/testmain",
"//testkit/testsetup",
"//types",
"//util",
"//util/collate",
"//util/hint",
"//util/logutil",
"//util/mock",
"//util/plancodec",
"@com_github_golang_snappy//:snappy",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_tipb//go-tipb",
"@com_github_stretchr_testify//require",
"@org_uber_go_goleak//:goleak",
],
)
Loading

0 comments on commit 0e2ba9e

Please sign in to comment.