From 0686013aa128f158039b4cb4d0038e0dc1633799 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Mon, 29 Aug 2022 23:24:39 +0800 Subject: [PATCH 01/13] refine stats loaded status and when to use pseudo stats --- planner/core/logical_plan_builder.go | 17 ++++--- statistics/column.go | 1 + statistics/handle/bootstrap.go | 18 ++++---- statistics/handle/handle.go | 67 ++++++++++++++++++---------- statistics/handle/handle_hist.go | 42 +++++++++-------- statistics/histogram.go | 10 +++++ statistics/table.go | 15 +++++++ 7 files changed, 114 insertions(+), 56 deletions(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 51358486b8c6f..13f55806d296d 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -4144,15 +4144,20 @@ func getStatsTable(ctx sessionctx.Context, tblInfo *model.TableInfo, pid int64) return statistics.PseudoTable(tblInfo) } - // 3. statistics is outdated. - if ctx.GetSessionVars().GetEnablePseudoForOutdatedStats() { - if statsTbl.IsOutdated() { - tbl := *statsTbl - tbl.Pseudo = true - statsTbl = &tbl + // 3. statistics is uninitialized or outdated. + pseudoStatsForUninitialized := !statsTbl.IsInitialized() + pseudoStatsForOutdated := ctx.GetSessionVars().GetEnablePseudoForOutdatedStats() && statsTbl.IsOutdated() + if pseudoStatsForUninitialized || pseudoStatsForOutdated { + tbl := *statsTbl + tbl.Pseudo = true + statsTbl = &tbl + if pseudoStatsForUninitialized { + pseudoEstimationNotAvailable.Inc() + } else { pseudoEstimationOutdate.Inc() } } + return statsTbl } diff --git a/statistics/column.go b/statistics/column.go index 197fc2f5a6794..f8f63e0556f2b 100644 --- a/statistics/column.go +++ b/statistics/column.go @@ -124,6 +124,7 @@ func (c *Column) IsInvalid(sctx sessionctx.Context, collPseudo bool) bool { if stmtctx != nil && stmtctx.StatsLoad.Fallback { return true } + // If stats is not initialized, do we still load it? if c.IsLoadNeeded() && stmtctx != nil { if stmtctx.StatsLoad.Timeout > 0 { logutil.BgLogger().Warn("Hist for column should already be loaded as sync but not found.", diff --git a/statistics/handle/bootstrap.go b/statistics/handle/bootstrap.go index 094a02bd1da79..07ad94d5143e9 100644 --- a/statistics/handle/bootstrap.go +++ b/statistics/handle/bootstrap.go @@ -112,14 +112,16 @@ func (h *Handle) initStatsHistograms4Chunk(is infoschema.InfoSchema, cache *stat } hist := statistics.NewHistogram(id, ndv, nullCount, version, types.NewFieldType(mysql.TypeBlob), chunk.InitialCapacity, 0) index := &statistics.Index{ - Histogram: *hist, - CMSketch: cms, - TopN: topN, - Info: idxInfo, - StatsVer: statsVer, - Flag: row.GetInt64(10), - PhysicalID: tblID, - StatsLoadedStatus: statistics.NewStatsFullLoadStatus(), + Histogram: *hist, + CMSketch: cms, + TopN: topN, + Info: idxInfo, + StatsVer: statsVer, + Flag: row.GetInt64(10), + PhysicalID: tblID, + } + if statsVer != statistics.Version0 { + index.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() } lastAnalyzePos.Copy(&index.LastAnalyzePos) table.Indices[hist.ID] = index diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 6c5ca30353176..68c85b58807a8 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -699,16 +699,19 @@ func (h *Handle) loadNeededColumnHistograms(reader *statsReader, col model.Table logutil.BgLogger().Error("fail to get stats version for this histogram", zap.Int64("table_id", col.TableID), zap.Int64("hist_id", col.ID)) return errors.Trace(fmt.Errorf("fail to get stats version for this histogram, table_id:%v, hist_id:%v", col.TableID, col.ID)) } + statsVer := rows[0].GetInt64(0) colHist := &statistics.Column{ - PhysicalID: col.TableID, - Histogram: *hg, - Info: c.Info, - CMSketch: cms, - TopN: topN, - FMSketch: fms, - IsHandle: c.IsHandle, - StatsVer: rows[0].GetInt64(0), - StatsLoadedStatus: statistics.NewStatsFullLoadStatus(), + PhysicalID: col.TableID, + Histogram: *hg, + Info: c.Info, + CMSketch: cms, + TopN: topN, + FMSketch: fms, + IsHandle: c.IsHandle, + StatsVer: statsVer, + } + if statsVer != statistics.Version0 { + colHist.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() } // Column.Count is calculated by Column.TotalRowCount(). Hence we don't set Column.Count when initializing colHist. colHist.Count = int64(colHist.TotalRowCount()) @@ -835,6 +838,7 @@ func (h *Handle) indexStatsFromStorage(reader *statsReader, row chunk.Row, table distinct := row.GetInt64(3) histVer := row.GetUint64(4) nullCount := row.GetInt64(5) + statsVer := row.GetInt64(7) idx := table.Indices[histID] errorRate := statistics.ErrorRate{} flag := row.GetInt64(8) @@ -861,10 +865,20 @@ func (h *Handle) indexStatsFromStorage(reader *statsReader, row chunk.Row, table if err != nil { return errors.Trace(err) } - idx = &statistics.Index{Histogram: *hg, CMSketch: cms, TopN: topN, FMSketch: fmSketch, - Info: idxInfo, ErrorRate: errorRate, StatsVer: row.GetInt64(7), Flag: flag, - PhysicalID: table.PhysicalID, - StatsLoadedStatus: statistics.NewStatsFullLoadStatus()} + idx = &statistics.Index{ + Histogram: *hg, + CMSketch: cms, + TopN: topN, + FMSketch: fmSketch, + Info: idxInfo, + ErrorRate: errorRate, + StatsVer: statsVer, + Flag: flag, + PhysicalID: table.PhysicalID, + } + if statsVer != statistics.Version0 { + idx.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() + } lastAnalyzePos.Copy(&idx.LastAnalyzePos) } break @@ -923,6 +937,9 @@ func (h *Handle) columnStatsFromStorage(reader *statsReader, row chunk.Row, tabl Flag: flag, StatsVer: statsVer, } + if statsVer != statistics.Version0 { + col.StatsLoadedStatus = statistics.NewStatsAllEvictedStatus() + } lastAnalyzePos.Copy(&col.LastAnalyzePos) col.Histogram.Correlation = correlation break @@ -943,17 +960,19 @@ func (h *Handle) columnStatsFromStorage(reader *statsReader, row chunk.Row, tabl return errors.Trace(err) } col = &statistics.Column{ - PhysicalID: table.PhysicalID, - Histogram: *hg, - Info: colInfo, - CMSketch: cms, - TopN: topN, - FMSketch: fmSketch, - ErrorRate: errorRate, - IsHandle: tableInfo.PKIsHandle && mysql.HasPriKeyFlag(colInfo.GetFlag()), - Flag: flag, - StatsVer: statsVer, - StatsLoadedStatus: statistics.NewStatsFullLoadStatus(), + PhysicalID: table.PhysicalID, + Histogram: *hg, + Info: colInfo, + CMSketch: cms, + TopN: topN, + FMSketch: fmSketch, + ErrorRate: errorRate, + IsHandle: tableInfo.PKIsHandle && mysql.HasPriKeyFlag(colInfo.GetFlag()), + Flag: flag, + StatsVer: statsVer, + } + if statsVer != statistics.Version0 { + col.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() } // Column.Count is calculated by Column.TotalRowCount(). Hence we don't set Column.Count when initializing col. col.Count = int64(col.TotalRowCount()) diff --git a/statistics/handle/handle_hist.go b/statistics/handle/handle_hist.go index 8e87308e22c64..28263e25e8a95 100644 --- a/statistics/handle/handle_hist.go +++ b/statistics/handle/handle_hist.go @@ -345,31 +345,37 @@ func (h *Handle) readStatsForOneItem(item model.TableItemID, w *statsWrapper, re zap.Int64("hist_id", item.ID), zap.Bool("is_index", item.IsIndex)) return nil, errors.Trace(fmt.Errorf("fail to get stats version for this histogram, table_id:%v, hist_id:%v, is_index:%v", item.TableID, item.ID, item.IsIndex)) } + statsVer := rows[0].GetInt64(0) if item.IsIndex { idxHist := &statistics.Index{ - Histogram: *hg, - CMSketch: cms, - TopN: topN, - FMSketch: fms, - Info: index.Info, - ErrorRate: index.ErrorRate, - StatsVer: rows[0].GetInt64(0), Flag: index.Flag, - PhysicalID: index.PhysicalID, - StatsLoadedStatus: statistics.NewStatsFullLoadStatus(), + Histogram: *hg, + CMSketch: cms, + TopN: topN, + FMSketch: fms, + Info: index.Info, + ErrorRate: index.ErrorRate, + StatsVer: statsVer, + Flag: index.Flag, + PhysicalID: index.PhysicalID, + } + if statsVer != statistics.Version0 { + idxHist.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() } index.LastAnalyzePos.Copy(&idxHist.LastAnalyzePos) w.idx = idxHist } else { colHist := &statistics.Column{ - PhysicalID: item.TableID, - Histogram: *hg, - Info: c.Info, - CMSketch: cms, - TopN: topN, - FMSketch: fms, - IsHandle: c.IsHandle, - StatsVer: rows[0].GetInt64(0), - StatsLoadedStatus: statistics.NewStatsFullLoadStatus(), + PhysicalID: item.TableID, + Histogram: *hg, + Info: c.Info, + CMSketch: cms, + TopN: topN, + FMSketch: fms, + IsHandle: c.IsHandle, + StatsVer: statsVer, + } + if statsVer != statistics.Version0 { + colHist.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() } // Column.Count is calculated by Column.TotalRowCount(). Hence, we don't set Column.Count when initializing colHist. colHist.Count = int64(colHist.TotalRowCount()) diff --git a/statistics/histogram.go b/statistics/histogram.go index 78db8f1b7259e..2133ccad3b53b 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -1570,6 +1570,16 @@ func NewStatsFullLoadStatus() StatsLoadedStatus { } } +// NewStatsAllEvictedStatus returns the status that only loads count/nullCount/NDV and doesn't load CMSketch/TopN/Histogram. +// When we load table stats, column stats is in allEvicted status by default. CMSketch/TopN/Histogram of column is only +// loaded when we really need column stats. +func NewStatsAllEvictedStatus() StatsLoadedStatus { + return StatsLoadedStatus{ + statsInitialized: true, + evictedStatus: allEvicted, + } +} + // IsStatsInitialized indicates whether the column/index's statistics was loaded from storage before. // Note that `IsStatsInitialized` only can be set in initializing func (s StatsLoadedStatus) IsStatsInitialized() bool { diff --git a/statistics/table.go b/statistics/table.go index 384bf3c4c62b0..960799b361e8d 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -463,6 +463,21 @@ func (n *neededStatsMap) Length() int { // and use pseudo estimation. var RatioOfPseudoEstimate = atomic.NewFloat64(0.7) +// IsInitialized returns true if any column/index stats of the table is initialized. +func (t *Table) IsInitialized() bool { + for _, col := range t.Columns { + if col != nil && col.IsStatsInitialized() { + return true + } + } + for _, idx := range t.Indices { + if idx != nil && idx.IsStatsInitialized() { + return true + } + } + return false +} + // IsOutdated returns true if the table stats is outdated. func (t *Table) IsOutdated() bool { rowcount := t.GetColRowCount() From a8ec66a14b6990235e6870fcfdd1537a1a34f1f4 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Tue, 30 Aug 2022 21:37:39 +0800 Subject: [PATCH 02/13] add test --- executor/show_stats.go | 3 +++ statistics/column.go | 1 - statistics/handle/handle_test.go | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/executor/show_stats.go b/executor/show_stats.go index 5f0adb235d68f..597c6923ccc6f 100644 --- a/executor/show_stats.go +++ b/executor/show_stats.go @@ -183,6 +183,9 @@ func (e *ShowExec) appendTableForStatsHistograms(dbName, tblName, partitionName col.StatsLoadedStatus.StatusToString(), col.MemoryUsage()) } for _, idx := range stableIdxsStats(statsTbl.Indices) { + if !idx.IsStatsInitialized() { + continue + } e.histogramToRow(dbName, tblName, partitionName, idx.Info.Name.O, 1, idx.Histogram, 0, idx.StatsLoadedStatus.StatusToString(), idx.MemoryUsage()) } diff --git a/statistics/column.go b/statistics/column.go index f8f63e0556f2b..197fc2f5a6794 100644 --- a/statistics/column.go +++ b/statistics/column.go @@ -124,7 +124,6 @@ func (c *Column) IsInvalid(sctx sessionctx.Context, collPseudo bool) bool { if stmtctx != nil && stmtctx.StatsLoad.Fallback { return true } - // If stats is not initialized, do we still load it? if c.IsLoadNeeded() && stmtctx != nil { if stmtctx.StatsLoad.Timeout > 0 { logutil.BgLogger().Warn("Hist for column should already be loaded as sync but not found.", diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index c2137802d0257..95a663949dd6d 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -3357,3 +3357,38 @@ func TestAnalyzeTableLRUPut(t *testing.T) { tk.MustExec("analyze table test.t") require.Equal(t, tbl.Meta().ID, domain.GetDomain(tk.Session()).StatsHandle().GetStatsCacheFrontTable()) } + +func TestUninitializedStatsStatus(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + dom.StatsHandle().SetLease(0) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b int, c int, index idx_a(a))") + h := dom.StatsHandle() + require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) + tk.MustExec("insert into t values (1,2,2), (3,4,4), (5,6,6), (7,8,8), (9,10,10)") + require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) + is := dom.InfoSchema() + require.NoError(t, h.Update(is)) + tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tblInfo := tbl.Meta() + tblStats := h.GetTableStats(tblInfo) + for _, col := range tblStats.Columns { + require.False(t, col.IsStatsInitialized()) + } + for _, idx := range tblStats.Indices { + require.False(t, idx.IsStatsInitialized()) + } + tk.MustQuery("show stats_histograms where db_name = 'test' and table_name = 't'").Check(testkit.Rows()) + checkStatsPseudo := func() { + rows := tk.MustQuery("explain select * from t").Rows() + operatorInfo := rows[len(rows)-1][4].(string) + require.True(t, strings.Contains(operatorInfo, "stats:pseudo")) + } + tk.MustExec("set @@tidb_enable_pseudo_for_outdated_stats = true") + checkStatsPseudo() + tk.MustExec("set @@tidb_enable_pseudo_for_outdated_stats = false") + checkStatsPseudo() +} From 71aeed2c6b89da682a2fe1b53640336c623c925e Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 2 Sep 2022 14:15:49 +0800 Subject: [PATCH 03/13] revise Signed-off-by: yisaer --- executor/builder.go | 18 +++++++++--------- executor/executor.go | 5 +++++ executor/show_stats.go | 10 +++++----- planner/core/access_object.go | 10 +++++----- planner/core/explain.go | 2 +- planner/core/find_best_task.go | 2 +- planner/core/integration_test.go | 26 ++++++++++++++++++++++++++ planner/core/logical_plan_builder.go | 11 ++++++++--- planner/core/plan_to_pb.go | 2 +- sessionctx/stmtctx/stmtctx.go | 7 +++++++ sessionctx/variable/session.go | 6 ++++-- 11 files changed, 72 insertions(+), 27 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index f87c57c92d162..ffaafa2041b79 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -3277,7 +3277,7 @@ func (b *executorBuilder) buildTableReader(v *plannercore.PhysicalTableReader) E sctx := b.ctx.GetSessionVars().StmtCtx sctx.TableIDs = append(sctx.TableIDs, ts.Table.ID) - if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if !b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return ret } // When isPartition is set, it means the union rewriting is done, so a partition reader is preferred. @@ -3481,7 +3481,7 @@ func buildNoRangeIndexReader(b *executorBuilder, v *plannercore.PhysicalIndexRea e.feedback = statistics.NewQueryFeedback(0, nil, 0, is.Desc) } else { tblID := e.physicalTableID - if b.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { tblID = e.table.Meta().ID } e.feedback = statistics.NewQueryFeedback(tblID, is.Hist, int64(is.StatsCount()), is.Desc) @@ -3524,7 +3524,7 @@ func (b *executorBuilder) buildIndexReader(v *plannercore.PhysicalIndexReader) E sctx := b.ctx.GetSessionVars().StmtCtx sctx.IndexNames = append(sctx.IndexNames, is.Table.Name.O+":"+is.Index.Name.O) - if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if !b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return ret } // When isPartition is set, it means the union rewriting is done, so a partition reader is preferred. @@ -3700,7 +3700,7 @@ func (b *executorBuilder) buildIndexLookUpReader(v *plannercore.PhysicalIndexLoo sctx.IndexNames = append(sctx.IndexNames, is.Table.Name.O+":"+is.Index.Name.O) sctx.TableIDs = append(sctx.TableIDs, ts.Table.ID) - if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if !b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return ret } @@ -3837,7 +3837,7 @@ func (b *executorBuilder) buildIndexMergeReader(v *plannercore.PhysicalIndexMerg sctx.TableIDs = append(sctx.TableIDs, ts.Table.ID) executorCounterIndexMergeReaderExecutor.Inc() - if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if !b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return ret } @@ -3955,7 +3955,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte return nil, err } tbInfo := e.table.Meta() - if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { if v.IsCommonHandle { kvRanges, err := buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { @@ -4203,7 +4203,7 @@ func (builder *dataReaderBuilder) buildIndexReaderForIndexJoin(ctx context.Conte return nil, err } tbInfo := e.table.Meta() - if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { kvRanges, err := buildKvRangesForIndexJoin(e.ctx, e.physicalTableID, e.index.ID, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memoryTracker, interruptSignal) if err != nil { return nil, err @@ -4250,7 +4250,7 @@ func (builder *dataReaderBuilder) buildIndexLookUpReaderForIndexJoin(ctx context } tbInfo := e.table.Meta() - if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { e.kvRanges, err = buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), e.index.ID, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { return nil, err @@ -4836,7 +4836,7 @@ func getPhysicalTableID(t table.Table) int64 { } func getFeedbackStatsTableID(ctx sessionctx.Context, t table.Table) int64 { - if p, ok := t.(table.PhysicalTable); ok && !ctx.GetSessionVars().UseDynamicPartitionPrune() { + if p, ok := t.(table.PhysicalTable); ok && !ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return p.GetPhysicalID() } return t.Meta().ID diff --git a/executor/executor.go b/executor/executor.go index e836ab6d522d3..57f424cee1157 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1927,6 +1927,11 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.OptimizerCETrace = nil sc.StatsLoadStatus = make(map[model.TableItemID]string) sc.IsSyncStatsFailed = false + // Firstly we assume that UseDynamicPruneMode can be enabled according session variable, then we will check other conditions + // in PlanBuilder.buildDataSource + if ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { + sc.UseDynamicPruneMode = true + } sc.SysdateIsNow = ctx.GetSessionVars().SysdateIsNow diff --git a/executor/show_stats.go b/executor/show_stats.go index 597c6923ccc6f..058b3ed7c62da 100644 --- a/executor/show_stats.go +++ b/executor/show_stats.go @@ -108,7 +108,7 @@ func (e *ShowExec) fetchShowStatsMeta() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { partitionName := "" if pi != nil { partitionName = "global" @@ -150,7 +150,7 @@ func (e *ShowExec) fetchShowStatsHistogram() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { partitionName := "" if pi != nil { partitionName = "global" @@ -224,7 +224,7 @@ func (e *ShowExec) fetchShowStatsBuckets() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { partitionName := "" if pi != nil { partitionName = "global" @@ -283,7 +283,7 @@ func (e *ShowExec) fetchShowStatsTopN() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { partitionName := "" if pi != nil { partitionName = "global" @@ -415,7 +415,7 @@ func (e *ShowExec) fetchShowStatsHealthy() { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { partitionName := "" if pi != nil { partitionName = "global" diff --git a/planner/core/access_object.go b/planner/core/access_object.go index 81296c39c9223..c57f7edc06b90 100644 --- a/planner/core/access_object.go +++ b/planner/core/access_object.go @@ -352,7 +352,7 @@ func (p *BatchPointGetPlan) AccessObject() AccessObject { func getDynamicAccessPartition(sctx sessionctx.Context, tblInfo *model.TableInfo, partitionInfo *PartitionInfo, asName string) (res *DynamicPartitionAccessObject) { pi := tblInfo.GetPartitionInfo() - if pi == nil || !sctx.GetSessionVars().UseDynamicPartitionPrune() { + if pi == nil || !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return nil } @@ -392,7 +392,7 @@ func getDynamicAccessPartition(sctx sessionctx.Context, tblInfo *model.TableInfo } func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) AccessObject { - if !sctx.GetSessionVars().UseDynamicPartitionPrune() { + if !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return DynamicPartitionAccessObjects(nil) } if len(p.PartitionInfos) == 0 { @@ -444,7 +444,7 @@ func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) AccessObject } func (p *PhysicalIndexReader) accessObject(sctx sessionctx.Context) AccessObject { - if !sctx.GetSessionVars().UseDynamicPartitionPrune() { + if !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return DynamicPartitionAccessObjects(nil) } is := p.IndexPlans[0].(*PhysicalIndexScan) @@ -460,7 +460,7 @@ func (p *PhysicalIndexReader) accessObject(sctx sessionctx.Context) AccessObject } func (p *PhysicalIndexLookUpReader) accessObject(sctx sessionctx.Context) AccessObject { - if !sctx.GetSessionVars().UseDynamicPartitionPrune() { + if !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return DynamicPartitionAccessObjects(nil) } ts := p.TablePlans[0].(*PhysicalTableScan) @@ -476,7 +476,7 @@ func (p *PhysicalIndexLookUpReader) accessObject(sctx sessionctx.Context) Access } func (p *PhysicalIndexMergeReader) accessObject(sctx sessionctx.Context) AccessObject { - if !sctx.GetSessionVars().UseDynamicPartitionPrune() { + if !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return DynamicPartitionAccessObjects(nil) } ts := p.TablePlans[0].(*PhysicalTableScan) diff --git a/planner/core/explain.go b/planner/core/explain.go index d056eebc37d60..62ece42048433 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -214,7 +214,7 @@ func (p *PhysicalTableScan) OperatorInfo(normalized bool) string { if p.stats.StatsVersion == statistics.PseudoVersion && !normalized { buffer.WriteString(", stats:pseudo") } - if p.StoreType == kv.TiFlash && p.Table.GetPartitionInfo() != nil && p.IsMPPOrBatchCop && p.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if p.StoreType == kv.TiFlash && p.Table.GetPartitionInfo() != nil && p.IsMPPOrBatchCop && p.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { buffer.WriteString(", PartitionTableScan:true") } return buffer.String() diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index f538e3fe62c8e..5a9b524373593 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -936,7 +936,7 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter var hashPartColName *ast.ColumnName if tblInfo := ds.table.Meta(); canConvertPointGet && tblInfo.GetPartitionInfo() != nil { // We do not build [batch] point get for dynamic table partitions now. This can be optimized. - if ds.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if ds.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { canConvertPointGet = false } if canConvertPointGet && len(path.Ranges) > 1 { diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index eade71c9bd3a4..6d005f382d210 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -7130,3 +7130,29 @@ func TestCastTimeAsDurationToTiFlash(t *testing.T) { } tk.MustQuery("explain select cast(a as time), cast(b as time) from t;").CheckAt([]int{0, 2, 4}, rows) } + +func TestPartitionTableFallBackPlan(t *testing.T) { + store, _ := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("set @@tidb_partition_prune_mode='static'") + tk.MustExec("CREATE TABLE t (a int) PARTITION BY RANGE (a) (PARTITION p0 VALUES LESS THAN (6),PARTITION p1 VALUES LESS THAN (11));") + tk.MustExec("insert into t values (1),(7)") + tk.MustExec("analyze table t") + rows := [][]interface{}{ + {"PartitionUnion_8", "root", "", ""}, + {"├─TableReader_10", "root", "", "data:TableFullScan_9"}, + {"│ └─TableFullScan_9", "cop[tikv]", "table:t, partition:p0", "keep order:false"}, + {"└─TableReader_12", "root", "", "data:TableFullScan_11"}, + {" └─TableFullScan_11", "cop[tikv]", "table:t, partition:p1", "keep order:false"}, + } + tk.MustQuery("explain select * from t").CheckAt([]int{0, 2, 3, 4}, rows) + tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") + tk.MustQuery("explain select * from t").CheckAt([]int{0, 2, 3, 4}, rows) + tk.MustExec("analyze table t") + rows = [][]interface{}{ + {"TableReader_5", "root", "partition:all", "data:TableFullScan_4"}, + {"└─TableFullScan_4", "cop[tikv]", "table:t", "keep order:false"}, + } + tk.MustQuery("explain select * from t").CheckAt([]int{0, 2, 3, 4}, rows) +} diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 13f55806d296d..ca73ae635c370 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -4132,7 +4132,7 @@ func getStatsTable(ctx sessionctx.Context, tblInfo *model.TableInfo, pid int64) } var statsTbl *statistics.Table - if pid == tblInfo.ID || ctx.GetSessionVars().UseDynamicPartitionPrune() { + if pid == tblInfo.ID || ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { statsTbl = statsHandle.GetTableStats(tblInfo, handle.WithTableStatsByQuery()) } else { statsTbl = statsHandle.GetPartitionStats(tblInfo, pid, handle.WithTableStatsByQuery()) @@ -4349,9 +4349,14 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as } if tableInfo.GetPartitionInfo() != nil { - // Use the new partition implementation, clean up the code here when it's full implemented. - if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { + h := domain.GetDomain(b.ctx).StatsHandle() + tblStats := h.GetTableStats(tableInfo) + isDynamicEnabled := b.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() + globalStatsReady := tblStats.IsInitialized() + // If dynamic partition prune isn't enabled or global stats is not ready, we won't enable dynamic prune mode in query + if !isDynamicEnabled || !globalStatsReady { b.optFlag = b.optFlag | flagPartitionProcessor + b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = false } pt := tbl.(table.PartitionedTable) diff --git a/planner/core/plan_to_pb.go b/planner/core/plan_to_pb.go index 0bd07a4415f20..59194e5b37286 100644 --- a/planner/core/plan_to_pb.go +++ b/planner/core/plan_to_pb.go @@ -185,7 +185,7 @@ func (p *PhysicalLimit) ToPB(ctx sessionctx.Context, storeType kv.StoreType) (*t // ToPB implements PhysicalPlan ToPB interface. func (p *PhysicalTableScan) ToPB(ctx sessionctx.Context, storeType kv.StoreType) (*tipb.Executor, error) { - if storeType == kv.TiFlash && p.Table.GetPartitionInfo() != nil && p.IsMPPOrBatchCop && p.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if storeType == kv.TiFlash && p.Table.GetPartitionInfo() != nil && p.IsMPPOrBatchCop && p.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { return p.partitionTableScanToPBForFlash(ctx) } tsExec := tables.BuildTableScanFromInfos(p.Table, p.Columns) diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index f62df41de3615..984ffcde64558 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -316,6 +316,8 @@ type StatementContext struct { StatsLoadStatus map[model.TableItemID]string // IsSyncStatsFailed indicates whether any failure happened during sync stats IsSyncStatsFailed bool + // UseDynamicPruneMode indicates whether use UseDynamicPruneMode in query stmt + UseDynamicPruneMode bool } // StmtHints are SessionVars related sql hints. @@ -981,6 +983,11 @@ func (sc *StatementContext) GetLockWaitStartTime() time.Time { return time.Unix(0, startTime) } +// UseDynamicPartitionPrune indicates whether dynamic partition is used during the query +func (sc *StatementContext) UseDynamicPartitionPrune() bool { + return sc.UseDynamicPruneMode +} + // CopTasksDetails collects some useful information of cop-tasks during execution. type CopTasksDetails struct { NumCopTasks int diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 0dc6eae68fdb8..03da2453e5c26 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1301,8 +1301,10 @@ func (s *SessionVars) CheckAndGetTxnScope() string { return kv.GlobalTxnScope } -// UseDynamicPartitionPrune indicates whether use new dynamic partition prune. -func (s *SessionVars) UseDynamicPartitionPrune() bool { +// IsDynamicPartitionPruneEnabled indicates whether dynamic partition prune enabled +// Note that: IsDynamicPartitionPruneEnabled only indicates whether dynamic partition prune mode is enabled according to +// session variable, it isn't guaranteed to be used during query due to other conditions checking. +func (s *SessionVars) IsDynamicPartitionPruneEnabled() bool { return PartitionPruneMode(s.PartitionPruneMode.Load()) == Dynamic } From 744b9ccf0c7bbb6c43b7db886c97b45db54f7a1f Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 2 Sep 2022 15:37:39 +0800 Subject: [PATCH 04/13] revise test Signed-off-by: yisaer --- executor/executor.go | 2 ++ planner/core/integration_test.go | 50 ++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/executor/executor.go b/executor/executor.go index 57f424cee1157..e59f13620cb51 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1931,6 +1931,8 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { // in PlanBuilder.buildDataSource if ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { sc.UseDynamicPruneMode = true + } else { + sc.UseDynamicPruneMode = false } sc.SysdateIsNow = ctx.GetSessionVars().SysdateIsNow diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 6d005f382d210..e976a98f8fe4b 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -7131,28 +7131,54 @@ func TestCastTimeAsDurationToTiFlash(t *testing.T) { tk.MustQuery("explain select cast(a as time), cast(b as time) from t;").CheckAt([]int{0, 2, 4}, rows) } -func TestPartitionTableFallBackPlan(t *testing.T) { +func TestPartitionTableFallBackStatic(t *testing.T) { store, _ := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("set @@tidb_partition_prune_mode='static'") tk.MustExec("CREATE TABLE t (a int) PARTITION BY RANGE (a) (PARTITION p0 VALUES LESS THAN (6),PARTITION p1 VALUES LESS THAN (11));") - tk.MustExec("insert into t values (1),(7)") + tk.MustExec("insert into t values (1),(2),(3),(4),(7),(8),(9),(10)") tk.MustExec("analyze table t") + + // use static plan in static mode rows := [][]interface{}{ - {"PartitionUnion_8", "root", "", ""}, - {"├─TableReader_10", "root", "", "data:TableFullScan_9"}, - {"│ └─TableFullScan_9", "cop[tikv]", "table:t, partition:p0", "keep order:false"}, - {"└─TableReader_12", "root", "", "data:TableFullScan_11"}, - {" └─TableFullScan_11", "cop[tikv]", "table:t, partition:p1", "keep order:false"}, + {"PartitionUnion_8", "", ""}, + {"├─TableReader_10", "", "data:TableFullScan_9"}, + {"│ └─TableFullScan_9", "table:t, partition:p0", "keep order:false"}, + {"└─TableReader_12", "", "data:TableFullScan_11"}, + {" └─TableFullScan_11", "table:t, partition:p1", "keep order:false"}, } - tk.MustQuery("explain select * from t").CheckAt([]int{0, 2, 3, 4}, rows) + tk.MustQuery("explain select * from t").CheckAt([]int{0, 3, 4}, rows) + + tk.MustExec("CREATE TABLE t2 (a int) PARTITION BY RANGE (a) (PARTITION p0 VALUES LESS THAN (6),PARTITION p1 VALUES LESS THAN (11));") + tk.MustExec("insert into t2 values (1),(2),(3),(4),(7),(8),(9),(10)") + tk.MustExec("analyze table t2") tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") - tk.MustQuery("explain select * from t").CheckAt([]int{0, 2, 3, 4}, rows) + + // use static plan in dynamic mode due to having not global stats + tk.MustQuery("explain select * from t").CheckAt([]int{0, 3, 4}, rows) tk.MustExec("analyze table t") + + // use dynamic plan in dynamic mode with global stats rows = [][]interface{}{ - {"TableReader_5", "root", "partition:all", "data:TableFullScan_4"}, - {"└─TableFullScan_4", "cop[tikv]", "table:t", "keep order:false"}, + {"TableReader_5", "partition:all", "data:TableFullScan_4"}, + {"└─TableFullScan_4", "table:t", "keep order:false"}, } - tk.MustQuery("explain select * from t").CheckAt([]int{0, 2, 3, 4}, rows) + tk.MustQuery("explain select * from t").CheckAt([]int{0, 3, 4}, rows) + + rows = [][]interface{}{ + {"Union_16", "", ""}, + {"├─PartitionUnion_18", "", ""}, + {"│ ├─TableReader_20", "", "data:TableFullScan_19"}, + {"│ │ └─TableFullScan_19", "table:t, partition:p0", "keep order:false"}, + {"│ └─TableReader_22", "", "data:TableFullScan_21"}, + {"│ └─TableFullScan_21", "table:t, partition:p1", "keep order:false"}, + {"└─PartitionUnion_24", "", ""}, + {" ├─TableReader_26", "", "data:TableFullScan_25"}, + {" │ └─TableFullScan_25", "table:t2, partition:p0", "keep order:false"}, + {" └─TableReader_28", "", "data:TableFullScan_27"}, + {" └─TableFullScan_27", "table:t2, partition:p1", "keep order:false"}, + } + // use static plan in dynamic mode due to t2 has no global stats + tk.MustQuery("explain select * from t union all select * from t2;").CheckAt([]int{0, 3, 4}, rows) } From a67066ab5bf27883c46f3a2eb6457818de6328c2 Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 2 Sep 2022 15:48:41 +0800 Subject: [PATCH 05/13] remove conflict code Signed-off-by: yisaer --- statistics/handle/handle.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 0249b77dca166..b285033b7f69d 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -710,9 +710,6 @@ func (h *Handle) loadNeededColumnHistograms(reader *statsReader, col model.Table IsHandle: c.IsHandle, StatsVer: statsVer, } - if statsVer != statistics.Version0 { - colHist.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() - } // Column.Count is calculated by Column.TotalRowCount(). Hence we don't set Column.Count when initializing colHist. colHist.Count = int64(colHist.TotalRowCount()) // When adding/modifying a column, we create its stats(all values are default values) without setting stats_ver. From 877e1433faab0356f594dc0b9f916ab7f2832482 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 7 Sep 2022 12:48:03 +0800 Subject: [PATCH 06/13] fix explain test Signed-off-by: yisaer --- cmd/explaintest/r/generated_columns.result | 1 + cmd/explaintest/r/select.result | 15 ++++++++------- cmd/explaintest/t/generated_columns.test | 2 ++ cmd/explaintest/t/select.test | 1 + 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/explaintest/r/generated_columns.result b/cmd/explaintest/r/generated_columns.result index a937079129e11..59311e829ad41 100644 --- a/cmd/explaintest/r/generated_columns.result +++ b/cmd/explaintest/r/generated_columns.result @@ -151,6 +151,7 @@ PARTITION p3 VALUES LESS THAN (4), PARTITION p4 VALUES LESS THAN (5), PARTITION p5 VALUES LESS THAN (6), PARTITION max VALUES LESS THAN MAXVALUE); +analyze table sgc3; EXPLAIN format = 'brief' SELECT * FROM sgc3 WHERE a <= 1; id estRows task access object operator info TableReader 3323.33 root partition:p0,p1 data:Selection diff --git a/cmd/explaintest/r/select.result b/cmd/explaintest/r/select.result index a31c03782bbfb..9577b28e47c35 100644 --- a/cmd/explaintest/r/select.result +++ b/cmd/explaintest/r/select.result @@ -381,19 +381,20 @@ PartitionUnion_8 20000.00 root └─TableReader_12 10000.00 root data:TableFullScan_11 └─TableFullScan_11 10000.00 cop[tikv] table:th, partition:p2 keep order:false, stats:pseudo set @@session.tidb_partition_prune_mode = 'dynamic'; +analyze table th; desc select * from th where a=-2; id estRows task access object operator info -TableReader_7 10.00 root partition:p2 data:Selection_6 -└─Selection_6 10.00 cop[tikv] eq(test.th.a, -2) - └─TableFullScan_5 10000.00 cop[tikv] table:th keep order:false, stats:pseudo +TableReader_7 1.00 root partition:p2 data:Selection_6 +└─Selection_6 1.00 cop[tikv] eq(test.th.a, -2) + └─TableFullScan_5 17.00 cop[tikv] table:th keep order:false desc select * from th; id estRows task access object operator info -TableReader_5 10000.00 root partition:all data:TableFullScan_4 -└─TableFullScan_4 10000.00 cop[tikv] table:th keep order:false, stats:pseudo +TableReader_5 17.00 root partition:all data:TableFullScan_4 +└─TableFullScan_4 17.00 cop[tikv] table:th keep order:false desc select * from th partition (p2,p1); id estRows task access object operator info -TableReader_5 10000.00 root partition:p1,p2 data:TableFullScan_4 -└─TableFullScan_4 10000.00 cop[tikv] table:th keep order:false, stats:pseudo +TableReader_5 17.00 root partition:p1,p2 data:TableFullScan_4 +└─TableFullScan_4 17.00 cop[tikv] table:th keep order:false set @@session.tidb_partition_prune_mode = DEFAULT; drop table if exists t; create table t(a int, b int); diff --git a/cmd/explaintest/t/generated_columns.test b/cmd/explaintest/t/generated_columns.test index c007f8ff42e66..48570e8ccbffb 100644 --- a/cmd/explaintest/t/generated_columns.test +++ b/cmd/explaintest/t/generated_columns.test @@ -107,6 +107,8 @@ PARTITION p4 VALUES LESS THAN (5), PARTITION p5 VALUES LESS THAN (6), PARTITION max VALUES LESS THAN MAXVALUE); +analyze table sgc3; + EXPLAIN format = 'brief' SELECT * FROM sgc3 WHERE a <= 1; EXPLAIN format = 'brief' SELECT * FROM sgc3 WHERE a < 7; diff --git a/cmd/explaintest/t/select.test b/cmd/explaintest/t/select.test index dbb505bb250f0..907066ffff4f4 100644 --- a/cmd/explaintest/t/select.test +++ b/cmd/explaintest/t/select.test @@ -190,6 +190,7 @@ desc select * from th where a=-2; desc select * from th; desc select * from th partition (p2,p1); set @@session.tidb_partition_prune_mode = 'dynamic'; +analyze table th; desc select * from th where a=-2; desc select * from th; desc select * from th partition (p2,p1); From 616d97a9948101d431ed2ff3eb1c24dba47adb3a Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 7 Sep 2022 14:52:49 +0800 Subject: [PATCH 07/13] fix explain test Signed-off-by: yisaer --- planner/core/integration_test.go | 44 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index bea5355c41611..15808d2a083b4 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -7142,13 +7142,13 @@ func TestPartitionTableFallBackStatic(t *testing.T) { // use static plan in static mode rows := [][]interface{}{ - {"PartitionUnion_8", "", ""}, - {"├─TableReader_10", "", "data:TableFullScan_9"}, - {"│ └─TableFullScan_9", "table:t, partition:p0", "keep order:false"}, - {"└─TableReader_12", "", "data:TableFullScan_11"}, - {" └─TableFullScan_11", "table:t, partition:p1", "keep order:false"}, + {"PartitionUnion", "", ""}, + {"├─TableReader", "", "data:TableFullScan"}, + {"│ └─TableFullScan", "table:t, partition:p0", "keep order:false"}, + {"└─TableReader", "", "data:TableFullScan"}, + {" └─TableFullScan", "table:t, partition:p1", "keep order:false"}, } - tk.MustQuery("explain select * from t").CheckAt([]int{0, 3, 4}, rows) + tk.MustQuery("explain format='brief' select * from t").CheckAt([]int{0, 3, 4}, rows) tk.MustExec("CREATE TABLE t2 (a int) PARTITION BY RANGE (a) (PARTITION p0 VALUES LESS THAN (6),PARTITION p1 VALUES LESS THAN (11));") tk.MustExec("insert into t2 values (1),(2),(3),(4),(7),(8),(9),(10)") @@ -7156,31 +7156,31 @@ func TestPartitionTableFallBackStatic(t *testing.T) { tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") // use static plan in dynamic mode due to having not global stats - tk.MustQuery("explain select * from t").CheckAt([]int{0, 3, 4}, rows) + tk.MustQuery("explain format='brief' select * from t").CheckAt([]int{0, 3, 4}, rows) tk.MustExec("analyze table t") // use dynamic plan in dynamic mode with global stats rows = [][]interface{}{ - {"TableReader_5", "partition:all", "data:TableFullScan_4"}, - {"└─TableFullScan_4", "table:t", "keep order:false"}, + {"TableReader", "partition:all", "data:TableFullScan"}, + {"└─TableFullScan", "table:t", "keep order:false"}, } - tk.MustQuery("explain select * from t").CheckAt([]int{0, 3, 4}, rows) + tk.MustQuery("explain format='brief' select * from t").CheckAt([]int{0, 3, 4}, rows) rows = [][]interface{}{ - {"Union_16", "", ""}, - {"├─PartitionUnion_18", "", ""}, - {"│ ├─TableReader_20", "", "data:TableFullScan_19"}, - {"│ │ └─TableFullScan_19", "table:t, partition:p0", "keep order:false"}, - {"│ └─TableReader_22", "", "data:TableFullScan_21"}, - {"│ └─TableFullScan_21", "table:t, partition:p1", "keep order:false"}, - {"└─PartitionUnion_24", "", ""}, - {" ├─TableReader_26", "", "data:TableFullScan_25"}, - {" │ └─TableFullScan_25", "table:t2, partition:p0", "keep order:false"}, - {" └─TableReader_28", "", "data:TableFullScan_27"}, - {" └─TableFullScan_27", "table:t2, partition:p1", "keep order:false"}, + {"Union", "", ""}, + {"├─PartitionUnion", "", ""}, + {"│ ├─TableReader", "", "data:TableFullScan"}, + {"│ │ └─TableFullScan", "table:t, partition:p0", "keep order:false"}, + {"│ └─TableReader", "", "data:TableFullScan"}, + {"│ └─TableFullScan", "table:t, partition:p1", "keep order:false"}, + {"└─PartitionUnion", "", ""}, + {" ├─TableReader", "", "data:TableFullScan"}, + {" │ └─TableFullScan", "table:t2, partition:p0", "keep order:false"}, + {" └─TableReader", "", "data:TableFullScan"}, + {" └─TableFullScan", "table:t2, partition:p1", "keep order:false"}, } // use static plan in dynamic mode due to t2 has no global stats - tk.MustQuery("explain select * from t union all select * from t2;").CheckAt([]int{0, 3, 4}, rows) + tk.MustQuery("explain format='brief' select * from t union all select * from t2;").CheckAt([]int{0, 3, 4}, rows) } func TestEnableTiFlashReadForWriteStmt(t *testing.T) { From 98bb4dcf1dd3fc10674d5c928e8d24ba00d1954a Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 7 Sep 2022 17:25:35 +0800 Subject: [PATCH 08/13] fix test Signed-off-by: yisaer --- ddl/db_partition_test.go | 5 ++++ executor/analyzetest/analyze_test.go | 2 ++ executor/distsql_test.go | 2 ++ executor/executor_issue_test.go | 2 ++ executor/index_lookup_join_test.go | 3 +++ executor/partition_table_test.go | 24 +++++++++++++++++++ executor/write_test.go | 3 +++ planner/cascades/integration_test.go | 3 +++ planner/core/cbo_test.go | 3 +++ .../core/collect_column_stats_usage_test.go | 3 +++ planner/core/integration_partition_test.go | 7 ++++++ planner/core/integration_test.go | 11 +++++++++ planner/core/logical_plan_builder.go | 13 +++++++++- planner/core/partition_pruner_test.go | 7 ++++++ planner/core/physical_plan_test.go | 6 +++++ planner/core/point_get_plan_test.go | 3 +++ planner/core/rule_join_reorder_test.go | 3 +++ statistics/handle/handle_test.go | 3 +++ statistics/integration_test.go | 2 ++ table/tables/partition_test.go | 3 +++ 20 files changed, 107 insertions(+), 1 deletion(-) diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index 609006d7d380d..ee831e739479c 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -656,6 +656,8 @@ create table log_message_1 ( } func TestPartitionRangeColumnsCollate(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("create schema PartitionRangeColumnsCollate") @@ -3569,6 +3571,9 @@ func TestPartitionListWithTimeType(t *testing.T) { } func TestPartitionListWithNewCollation(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") + store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test;") diff --git a/executor/analyzetest/analyze_test.go b/executor/analyzetest/analyze_test.go index f544cbd0983cb..0ed5b6a53cbbd 100644 --- a/executor/analyzetest/analyze_test.go +++ b/executor/analyzetest/analyze_test.go @@ -2833,6 +2833,8 @@ PARTITION BY RANGE ( a ) ( } func TestAnalyzePartitionStaticToDynamic(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) diff --git a/executor/distsql_test.go b/executor/distsql_test.go index a0fc642a020d7..1f895a6703420 100644 --- a/executor/distsql_test.go +++ b/executor/distsql_test.go @@ -291,6 +291,8 @@ func TestPushLimitDownIndexLookUpReader(t *testing.T) { } func TestPartitionTableIndexLookUpReader(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/executor/executor_issue_test.go b/executor/executor_issue_test.go index 8e8b4d28f0fa3..febdbd6031823 100644 --- a/executor/executor_issue_test.go +++ b/executor/executor_issue_test.go @@ -583,6 +583,8 @@ func TestFix31537(t *testing.T) { } func TestIssue30382(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index dfd9f04a69140..24168058353ec 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/testkit" "github.com/stretchr/testify/require" ) @@ -391,6 +392,8 @@ func TestIssue24547(t *testing.T) { } func TestIssue27138(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index ce553e7b29f11..03fbc331ca21c 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" @@ -33,6 +34,8 @@ import ( ) func TestFourReader(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -529,6 +532,8 @@ func TestView(t *testing.T) { } func TestDirectReadingwithIndexJoin(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -641,6 +646,8 @@ func TestDirectReadingwithIndexJoin(t *testing.T) { } func TestDynamicPruningUnderIndexJoin(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -943,6 +950,8 @@ func TestGlobalStatsAndSQLBinding(t *testing.T) { } func TestPartitionTableWithDifferentJoin(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -1764,6 +1773,8 @@ func TestAddDropPartitions(t *testing.T) { } func TestMPPQueryExplainInfo(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -1793,6 +1804,8 @@ func TestMPPQueryExplainInfo(t *testing.T) { } func TestPartitionPruningInTransaction(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2026,6 +2039,8 @@ func TestSubqueries(t *testing.T) { } func TestSplitRegion(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2062,6 +2077,9 @@ func TestSplitRegion(t *testing.T) { } func TestParallelApply(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") + store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2569,6 +2587,8 @@ func TestDirectReadingWithAgg(t *testing.T) { } func TestDynamicModeByDefault(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2764,6 +2784,8 @@ func TestIssue25309(t *testing.T) { } func TestGlobalIndexScan(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2784,6 +2806,8 @@ partition p2 values less than (10))`) } func TestGlobalIndexDoubleRead(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/executor/write_test.go b/executor/write_test.go index 0c28d521c61a1..c99a3eebee5ee 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -21,6 +21,7 @@ import ( "strconv" "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" @@ -4200,6 +4201,8 @@ func TestUpdate(t *testing.T) { } func TestListColumnsPartitionWithGlobalIndex(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/planner/cascades/integration_test.go b/planner/cascades/integration_test.go index e3a7b750373ec..ebb80b192a383 100644 --- a/planner/cascades/integration_test.go +++ b/planner/cascades/integration_test.go @@ -18,6 +18,7 @@ import ( "fmt" "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/planner/cascades" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/testkit" @@ -375,6 +376,8 @@ func TestTopN(t *testing.T) { } func TestCascadePlannerHashedPartTable(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index d8d46c04d2aaa..31ba6bfeb3e07 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -23,6 +23,7 @@ import ( "strings" "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/parser/model" @@ -819,6 +820,8 @@ func TestLimitIndexEstimation(t *testing.T) { } func TestBatchPointGetTablePartition(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) testKit := testkit.NewTestKit(t, store) testKit.MustExec("use test") diff --git a/planner/core/collect_column_stats_usage_test.go b/planner/core/collect_column_stats_usage_test.go index d494235bc66fc..38d246ff8bfd7 100644 --- a/planner/core/collect_column_stats_usage_test.go +++ b/planner/core/collect_column_stats_usage_test.go @@ -20,6 +20,7 @@ import ( "sort" "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/util/hint" @@ -272,6 +273,8 @@ func TestCollectPredicateColumns(t *testing.T) { } func TestCollectHistNeededColumns(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") tests := []struct { pruneMode string sql string diff --git a/planner/core/integration_partition_test.go b/planner/core/integration_partition_test.go index e8c53739d2819..353fc379be794 100644 --- a/planner/core/integration_partition_test.go +++ b/planner/core/integration_partition_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/parser/auth" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" @@ -61,6 +62,9 @@ func TestListPartitionPushDown(t *testing.T) { } func TestListColVariousTypes(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") + store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -102,6 +106,9 @@ func TestListColVariousTypes(t *testing.T) { } func TestListPartitionPruning(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") + store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 15808d2a083b4..1b91cc8106c7e 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1452,6 +1452,8 @@ func TestPartitionTableStats(t *testing.T) { } func TestPartitionPruningForInExpr(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -1477,6 +1479,8 @@ func TestPartitionPruningForInExpr(t *testing.T) { } func TestPartitionPruningWithDateType(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -3375,6 +3379,8 @@ func TestExplainAnalyzeDML2(t *testing.T) { } func TestPartitionExplain(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") @@ -3549,6 +3555,8 @@ func TestPartitionUnionWithPPruningColumn(t *testing.T) { } func TestIssue20139(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -6519,6 +6527,9 @@ func TestIssue32632(t *testing.T) { } func TestTiFlashPartitionTableScan(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") + store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index bc8ac09e4c18f..1e75abca44e3e 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -17,6 +17,7 @@ package core import ( "context" "fmt" + "github.com/pingcap/failpoint" "math" "math/bits" "sort" @@ -4379,7 +4380,17 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as isDynamicEnabled := b.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() globalStatsReady := tblStats.IsInitialized() // If dynamic partition prune isn't enabled or global stats is not ready, we won't enable dynamic prune mode in query - if !isDynamicEnabled || !globalStatsReady { + usePartitionProcessor := !isDynamicEnabled || !globalStatsReady + + failpoint.Inject("forceDynamicPrune", func(val failpoint.Value) { + if val.(bool) { + if isDynamicEnabled { + usePartitionProcessor = false + } + } + }) + + if usePartitionProcessor { b.optFlag = b.optFlag | flagPartitionProcessor b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = false } diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go index d3db641a8efd9..bd78b1f370574 100644 --- a/planner/core/partition_pruner_test.go +++ b/planner/core/partition_pruner_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "github.com/pingcap/failpoint" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/testkit" @@ -29,6 +30,8 @@ import ( ) func TestHashPartitionPruner(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("create database test_partition") @@ -250,6 +253,8 @@ func TestRangeColumnPartitionPruningForInString(t *testing.T) { } func TestListPartitionPruner(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("drop database if exists test_partition;") @@ -321,6 +326,8 @@ func TestListPartitionPruner(t *testing.T) { } func TestListColumnsPartitionPruner(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("set @@session.tidb_enable_list_partition = ON") diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 61a2f5dd87e11..d4cf5a5a10430 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/infoschema" @@ -1259,6 +1260,8 @@ func doTestPushdownDistinct(t *testing.T, vars, input []string, output []struct } func TestGroupConcatOrderby(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") var ( input []string output []struct { @@ -2066,6 +2069,9 @@ func TestHJBuildAndProbeHint4StaticPartitionTable(t *testing.T) { } func TestHJBuildAndProbeHint4DynamicPartitionTable(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") + var ( input []string output []struct { diff --git a/planner/core/point_get_plan_test.go b/planner/core/point_get_plan_test.go index 71804ae0fc00e..3a359ddb18f47 100644 --- a/planner/core/point_get_plan_test.go +++ b/planner/core/point_get_plan_test.go @@ -17,6 +17,7 @@ package core_test import ( "context" "fmt" + "github.com/pingcap/failpoint" "strings" "testing" "time" @@ -653,6 +654,8 @@ func TestBatchPointGetPartition(t *testing.T) { } func TestBatchPointGetPartitionForAccessObject(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/rule_join_reorder_test.go b/planner/core/rule_join_reorder_test.go index a981495807d3d..a0c136d8d4a2f 100644 --- a/planner/core/rule_join_reorder_test.go +++ b/planner/core/rule_join_reorder_test.go @@ -17,6 +17,7 @@ package core_test import ( "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/parser/model" plannercore "github.com/pingcap/tidb/planner/core" @@ -233,6 +234,8 @@ func TestJoinOrderHint4StaticPartitionTable(t *testing.T) { } func TestJoinOrderHint4DynamicPartitionTable(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 95a663949dd6d..5e61f0f75bd83 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -2121,6 +2121,9 @@ func TestAnalyzeWithDynamicPartitionPruneMode(t *testing.T) { } func TestPartitionPruneModeSessionVariable(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") + store := testkit.CreateMockStore(t) tk1 := testkit.NewTestKit(t, store) tk1.MustExec("use test") diff --git a/statistics/integration_test.go b/statistics/integration_test.go index 5abba85ff9193..c4e4d315c7dbe 100644 --- a/statistics/integration_test.go +++ b/statistics/integration_test.go @@ -300,6 +300,8 @@ func TestExpBackoffEstimation(t *testing.T) { } func TestGlobalStats(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/table/tables/partition_test.go b/table/tables/partition_test.go index b05cf8f5037bd..aed5ef219ad87 100644 --- a/table/tables/partition_test.go +++ b/table/tables/partition_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" + "github.com/pingcap/failpoint" mysql "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/model" @@ -393,6 +394,8 @@ func TestLocatePartitionSingleColumn(t *testing.T) { } func TestLocatePartition(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") From c99442a42561ad1b5a27c3ef029c0639fc9842ff Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 7 Sep 2022 17:36:25 +0800 Subject: [PATCH 09/13] fix test Signed-off-by: yisaer --- planner/core/logical_plan_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 1e75abca44e3e..b632d352956bf 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -17,7 +17,6 @@ package core import ( "context" "fmt" - "github.com/pingcap/failpoint" "math" "math/bits" "sort" @@ -27,6 +26,7 @@ import ( "unicode" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" From 3a24a8bd73b7294b4391596eec455831ca208c6a Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 7 Sep 2022 17:54:22 +0800 Subject: [PATCH 10/13] fix test Signed-off-by: yisaer --- planner/core/point_get_plan_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/point_get_plan_test.go b/planner/core/point_get_plan_test.go index 3a359ddb18f47..3a92d25719c09 100644 --- a/planner/core/point_get_plan_test.go +++ b/planner/core/point_get_plan_test.go @@ -17,11 +17,11 @@ package core_test import ( "context" "fmt" - "github.com/pingcap/failpoint" "strings" "testing" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/planner" From 6f11717c452f959d0d97d7700cccb050d3f2359f Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 7 Sep 2022 18:35:49 +0800 Subject: [PATCH 11/13] fix test Signed-off-by: yisaer --- executor/analyzetest/analyze_test.go | 2 +- executor/partition_table_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/executor/analyzetest/analyze_test.go b/executor/analyzetest/analyze_test.go index 0ed5b6a53cbbd..b9bfc153fa33c 100644 --- a/executor/analyzetest/analyze_test.go +++ b/executor/analyzetest/analyze_test.go @@ -2903,7 +2903,7 @@ PARTITION BY RANGE ( a ) ( tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") require.NoError(t, h.LoadNeededHistograms()) tbl := h.GetTableStats(tableInfo) - require.Equal(t, 0, len(tbl.Columns)) + require.Equal(t, 4, len(tbl.Columns)) // ignore both p0's 3 buckets, persisted-partition-options' 1 bucket, just use table-level 2 buckets tk.MustExec("analyze table t partition p0") diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index 03fbc331ca21c..72fbf75c377bf 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -1740,6 +1740,8 @@ func TestDynamicPruneModeWithExpression(t *testing.T) { } func TestAddDropPartitions(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) From d29d7951e7226e438dd573bae8bbb76bd9cdc963 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 8 Sep 2022 00:14:48 +0800 Subject: [PATCH 12/13] fix plan Signed-off-by: yisaer --- executor/tiflashtest/tiflash_test.go | 2 ++ executor/write_test.go | 2 ++ planner/core/partition_pruner_test.go | 2 ++ 3 files changed, 6 insertions(+) diff --git a/executor/tiflashtest/tiflash_test.go b/executor/tiflashtest/tiflash_test.go index 5725ca22df4e2..d18dc2c87132c 100644 --- a/executor/tiflashtest/tiflash_test.go +++ b/executor/tiflashtest/tiflash_test.go @@ -457,6 +457,8 @@ func TestTiFlashPartitionTableReader(t *testing.T) { } func TestPartitionTable(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t, withMockTiFlash(2)) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/executor/write_test.go b/executor/write_test.go index c99a3eebee5ee..ecd33e4666c6a 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -3238,6 +3238,8 @@ func TestWriteListPartitionTable1(t *testing.T) { // TestWriteListPartitionTable2 test for write list partition when the partition expression is complicated and contain generated column. func TestWriteListPartitionTable2(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go index bd78b1f370574..fffef54373bd8 100644 --- a/planner/core/partition_pruner_test.go +++ b/planner/core/partition_pruner_test.go @@ -539,6 +539,8 @@ func TestListColumnsPartitionPrunerRandom(t *testing.T) { } func TestIssue22635(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("USE test;") From 2c4f5b0899d5cf63dc14699b896e394b20b69682 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 8 Sep 2022 00:33:42 +0800 Subject: [PATCH 13/13] fix plan Signed-off-by: yisaer --- executor/write_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/executor/write_test.go b/executor/write_test.go index ecd33e4666c6a..a5ef9da73ce00 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -3362,6 +3362,8 @@ func TestWriteListPartitionTable2(t *testing.T) { } func TestWriteListColumnsPartitionTable1(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test")