fix conflicts

pkg/statistics/handle/storage/read.go

@@ -22,6 +22,7 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/pkg/config"
+	"github.com/pingcap/tidb/pkg/infoschema"
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/parser/ast"
 	"github.com/pingcap/tidb/pkg/parser/model"
@@ -576,7 +577,7 @@ func LoadHistogram(sctx sessionctx.Context, tableID int64, isIndex int, histID i
 }
 
 // LoadNeededHistograms will load histograms for those needed columns/indices.
-func LoadNeededHistograms(sctx sessionctx.Context, statsCache statstypes.StatsCache, loadFMSketch bool) (err error) {
+func LoadNeededHistograms(sctx sessionctx.Context, statsCache statstypes.StatsHandle, loadFMSketch bool) (err error) {
 	items := statistics.HistogramNeededItems.AllItems()
 	for _, item := range items {
 		if !item.IsIndex {
@@ -619,54 +620,43 @@ func CleanFakeItemsForShowHistInFlights(statsCache statstypes.StatsCache) int {
 	return reallyNeeded
 }
 
-<<<<<<< HEAD
-func loadNeededColumnHistograms(sctx sessionctx.Context, statsCache statstypes.StatsCache, col model.TableItemID, loadFMSketch bool, fullLoad bool) (err error) {
-	tbl, ok := statsCache.Get(col.TableID)
+func loadNeededColumnHistograms(sctx sessionctx.Context, statsCache statstypes.StatsHandle, col model.TableItemID, loadFMSketch bool, fullLoad bool) (err error) {
+	statsTbl, ok := statsCache.Get(col.TableID)
 	if !ok {
 		return nil
 	}
-	var colInfo *model.ColumnInfo
-	_, loadNeeded, analyzed := tbl.ColumnIsLoadNeeded(col.ID, true)
-	if !loadNeeded || !analyzed {
-		statistics.HistogramNeededItems.Delete(col)
-		return nil
-	}
-	colInfo = tbl.ColAndIdxExistenceMap.GetCol(col.ID)
-=======
-func loadNeededColumnHistograms(sctx sessionctx.Context, statsHandle statstypes.StatsHandle, col model.TableItemID, loadFMSketch bool, fullLoad bool) (err error) {
-	statsTbl, ok := statsHandle.Get(col.TableID)
-	if !ok {
-		return nil
-	}
-	// Now, we cannot init the column info in the ColAndIdxExistenceMap when to disable lite-init-stats.
-	// so we have to get the column info from the domain.
 	is := sctx.GetDomainInfoSchema().(infoschema.InfoSchema)
-	tbl, ok := statsHandle.TableInfoByID(is, col.TableID)
+	tbl, ok := statsCache.TableInfoByID(is, col.TableID)
 	if !ok {
 		return nil
 	}
 	tblInfo := tbl.Meta()
-	colInfo := tblInfo.GetColumnByID(col.ID)
+	var colInfo *model.ColumnInfo
+	_, loadNeeded, analyzed := statsTbl.ColumnIsLoadNeeded(col.ID, true)
+	for _, col := range tblInfo.Columns {
+		if col.ID == col.ID {
+			colInfo = col
+			break
+		}
+	}
 	if colInfo == nil {
-		asyncload.AsyncLoadHistogramNeededItems.Delete(col)
+		statistics.HistogramNeededItems.Delete(col)
 		return nil
 	}
-
-	_, loadNeeded, analyzed := statsTbl.ColumnIsLoadNeeded(col.ID, true)
 	if !loadNeeded || !analyzed {
 		// If this column is not analyzed yet and we don't have it in memory.
 		// We create a fake one for the pseudo estimation.
 		// Otherwise, it will trigger the sync/async load again, even if the column has not been analyzed.
 		if loadNeeded && !analyzed {
-			fakeCol := statistics.EmptyColumn(tblInfo.ID, tblInfo.PKIsHandle, colInfo)
-			statsTbl.SetCol(col.ID, fakeCol)
-			statsHandle.UpdateStatsCache([]*statistics.Table{statsTbl}, nil)
+
+			fakeCol := statistics.EmptyColumn(colInfo.ID, tblInfo.PKIsHandle, colInfo)
+			statsTbl.Columns[col.ID] = fakeCol
+			statsCache.UpdateStatsCache([]*statistics.Table{statsTbl}, nil)
 		}
-		asyncload.AsyncLoadHistogramNeededItems.Delete(col)
+		statistics.HistogramNeededItems.Delete(col)
 		return nil
 	}
 
->>>>>>> 2b03447f198 (statistics: fix some problem related to stats async load (#57723))
 	hg, _, statsVer, _, err := HistMetaFromStorageWithHighPriority(sctx, &col, colInfo)
 	if hg == nil || err != nil {
 		statistics.HistogramNeededItems.Delete(col)
@@ -700,20 +690,12 @@ func loadNeededColumnHistograms(sctx sessionctx.Context, statsHandle statstypes.
 		CMSketch:   cms,
 		TopN:       topN,
 		FMSketch:   fms,
-<<<<<<< HEAD
-		IsHandle:   tbl.IsPkIsHandle && mysql.HasPriKeyFlag(colInfo.GetFlag()),
-=======
 		IsHandle:   tblInfo.PKIsHandle && mysql.HasPriKeyFlag(colInfo.GetFlag()),
->>>>>>> 2b03447f198 (statistics: fix some problem related to stats async load (#57723))
 		StatsVer:   statsVer,
 	}
 	// Reload the latest stats cache, otherwise the `updateStatsCache` may fail with high probability, because functions
 	// like `GetPartitionStats` called in `fmSketchFromStorage` would have modified the stats cache already.
-<<<<<<< HEAD
-	tbl, ok = statsCache.Get(col.TableID)
-=======
-	statsTbl, ok = statsHandle.Get(col.TableID)
->>>>>>> 2b03447f198 (statistics: fix some problem related to stats async load (#57723))
+	statsTbl, ok = statsCache.Get(col.TableID)
 	if !ok {
 		return nil
 	}
@@ -729,15 +711,9 @@ func loadNeededColumnHistograms(sctx sessionctx.Context, statsHandle statstypes.
 			statsTbl.StatsVer = int(statsVer)
 		}
 	}
-<<<<<<< HEAD
-	tbl.Columns[col.ID] = colHist
-	statsCache.UpdateStatsCache([]*statistics.Table{tbl}, nil)
+	statsTbl.Columns[col.ID] = colHist
+	statsCache.UpdateStatsCache([]*statistics.Table{statsTbl}, nil)
 	statistics.HistogramNeededItems.Delete(col)
-=======
-	statsTbl.SetCol(col.ID, colHist)
-	statsHandle.UpdateStatsCache([]*statistics.Table{statsTbl}, nil)
-	asyncload.AsyncLoadHistogramNeededItems.Delete(col)
->>>>>>> 2b03447f198 (statistics: fix some problem related to stats async load (#57723))
 	if col.IsSyncLoadFailed {
 		logutil.BgLogger().Warn("Hist for column should already be loaded as sync but not found.",
 			zap.Int64("table_id", colHist.PhysicalID),
@@ -793,14 +769,8 @@ func loadNeededIndexHistograms(sctx sessionctx.Context, statsCache statstypes.St
 		tbl.StatsVer = int(idxHist.StatsVer)
 		tbl.LastAnalyzeVersion = max(tbl.LastAnalyzeVersion, idxHist.LastUpdateVersion)
 	}
-<<<<<<< HEAD
 	tbl.Indices[idx.ID] = idxHist
-	tbl.LastAnalyzeVersion = max(tbl.LastAnalyzeVersion, idxHist.LastUpdateVersion)
 	statsCache.UpdateStatsCache([]*statistics.Table{tbl}, nil)
-=======
-	tbl.SetIdx(idx.ID, idxHist)
-	statsHandle.UpdateStatsCache([]*statistics.Table{tbl}, nil)
->>>>>>> 2b03447f198 (statistics: fix some problem related to stats async load (#57723))
 	if idx.IsSyncLoadFailed {
 		logutil.BgLogger().Warn("Hist for column should already be loaded as sync but not found.",
 			zap.Int64("table_id", idx.TableID),

pkg/statistics/handle/syncload/stats_syncload.go

@@ -312,18 +312,8 @@ func (s *statsSyncLoad) handleOneItemTask(task *statstypes.NeededItemTask) (err
 		// We create a fake one for the pseudo estimation.
 		// Otherwise, it will trigger the sync/async load again, even if the column has not been analyzed.
 		if loadNeeded && !analyzed {
-<<<<<<< HEAD
-			wrapper.col = &statistics.Column{
-				PhysicalID: item.TableID,
-				Info:       wrapper.colInfo,
-				Histogram:  *statistics.NewHistogram(item.ID, 0, 0, 0, &wrapper.colInfo.FieldType, 0, 0),
-				IsHandle:   tbl.IsPkIsHandle && mysql.HasPriKeyFlag(wrapper.colInfo.GetFlag()),
-			}
+			wrapper.col = statistics.EmptyColumn(item.TableID, tbl.IsPkIsHandle, wrapper.colInfo)
 			s.updateCachedItem(item, wrapper.col, wrapper.idx, task.Item.FullLoad)
-=======
-			wrapper.col = statistics.EmptyColumn(item.TableID, isPkIsHandle, wrapper.colInfo)
-			s.updateCachedItem(tblInfo, item, wrapper.col, wrapper.idx, task.Item.FullLoad)
->>>>>>> 2b03447f198 (statistics: fix some problem related to stats async load (#57723))
 			return nil
 		}
 	}

pkg/statistics/integration_test.go

@@ -574,35 +574,6 @@ func TestTableLastAnalyzeVersion(t *testing.T) {
 	require.True(t, found)
 	require.NotEqual(t, uint64(0), statsTbl.LastAnalyzeVersion)
 }
-<<<<<<< HEAD
-=======
-
-func TestGlobalIndexWithAnalyzeVersion1AndHistoricalStats(t *testing.T) {
-	store, dom := testkit.CreateMockStoreAndDomain(t)
-	tk := testkit.NewTestKit(t, store)
-
-	tk.MustExec("set tidb_analyze_version = 1")
-	tk.MustExec("set global tidb_enable_historical_stats = true")
-	defer tk.MustExec("set global tidb_enable_historical_stats = default")
-
-	tk.MustExec("use test")
-	tk.MustExec(`CREATE TABLE t ( a int, b int, c int default 0)
-					PARTITION BY RANGE (a) (
-					PARTITION p0 VALUES LESS THAN (10),
-					PARTITION p1 VALUES LESS THAN (20),
-					PARTITION p2 VALUES LESS THAN (30),
-					PARTITION p3 VALUES LESS THAN (40))`)
-	tk.MustExec("ALTER TABLE t ADD UNIQUE INDEX idx(b) GLOBAL")
-	tk.MustExec("INSERT INTO t(a, b) values(1, 1), (2, 2), (3, 3), (15, 15), (25, 25), (35, 35)")
-
-	tblID := dom.MustGetTableID(t, "test", "t")
-
-	for i := 0; i < 10; i++ {
-		tk.MustExec("analyze table t")
-	}
-	// Each analyze will only generate one record
-	tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id=%d", tblID)).Equal(testkit.Rows("10"))
-}
 
 func TestLastAnalyzeVersionNotChangedWithAsyncStatsLoad(t *testing.T) {
 	store, dom := testkit.CreateMockStoreAndDomain(t)
@@ -612,17 +583,16 @@ func TestLastAnalyzeVersionNotChangedWithAsyncStatsLoad(t *testing.T) {
 	tk.MustExec("use test")
 	tk.MustExec("create table t(a int, b int);")
 	require.NoError(t, dom.StatsHandle().HandleDDLEvent(<-dom.StatsHandle().DDLEventCh()))
-	require.NoError(t, dom.StatsHandle().Update(context.Background(), dom.InfoSchema()))
+	require.NoError(t, dom.StatsHandle().Update(dom.InfoSchema()))
 	tk.MustExec("insert into t values (1, 1);")
 	err := dom.StatsHandle().DumpStatsDeltaToKV(true)
 	require.NoError(t, err)
 	tk.MustExec("alter table t add column c int default 1;")
 	dom.StatsHandle().HandleDDLEvent(<-dom.StatsHandle().DDLEventCh())
 	tk.MustExec("select * from t where a = 1 or b = 1 or c = 1;")
-	require.NoError(t, dom.StatsHandle().LoadNeededHistograms(dom.InfoSchema()))
+	require.NoError(t, dom.StatsHandle().LoadNeededHistograms())
 	result := tk.MustQuery("show stats_meta where table_name = 't'")
 	require.Len(t, result.Rows(), 1)
 	// The last analyze time.
 	require.Equal(t, "<nil>", result.Rows()[0][6])
 }
->>>>>>> 2b03447f198 (statistics: fix some problem related to stats async load (#57723))

pkg/statistics/table.go

@@ -652,21 +652,10 @@ func (t *Table) ColumnIsLoadNeeded(id int64, fullLoad bool) (*Column, bool, bool
 	if t.Pseudo {
 		return nil, false, false
 	}
-<<<<<<< HEAD
 	col, ok := t.Columns[id]
-=======
-	// when we use non-lite init stats, it cannot init the stats for common columns.
-	// so we need to force to load the stats.
-	col, ok := t.columns[id]
-	if !ok {
-		return nil, true, true
-	}
->>>>>>> 2b03447f198 (statistics: fix some problem related to stats async load (#57723))
 	hasAnalyzed := t.ColAndIdxExistenceMap.HasAnalyzed(id, false)
 
 	// If it's not analyzed yet.
-	// The real check condition: !ok && !hashAnalyzed.
-	// After this check, we will always have ok && hasAnalyzed.
 	if !hasAnalyzed {
 		// If we don't have it in memory, we create a fake hist for pseudo estimation (see handleOneItemTask()).
 		if !ok {
@@ -680,9 +669,10 @@ func (t *Table) ColumnIsLoadNeeded(id int64, fullLoad bool) (*Column, bool, bool
 	}
 
 	// Restore the condition from the simplified form:
-	// 1. ok && hasAnalyzed && fullLoad && !col.IsFullLoad => need load
-	// 2. ok && hasAnalyzed && !fullLoad && !col.statsInitialized => need load
-	if (fullLoad && !col.IsFullLoad()) || (!fullLoad && !col.statsInitialized) {
+	// 1. !ok && hasAnalyzed => need load
+	// 2. ok && hasAnalyzed && fullLoad && !col.IsFullLoad => need load
+	// 3. ok && hasAnalyzed && !fullLoad && !col.statsInitialized => need load
+	if !ok || (fullLoad && !col.IsFullLoad()) || (!fullLoad && !col.statsInitialized) {
 		return col, true, true
 	}