Merge branch '2.0-dev' into relation-exists-2.0

matrixorigin · Nov 28, 2024 · aa1daeb · aa1daeb
2 parents e1a3600 + 1f3926e
commit aa1daeb
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 10 deletions.
diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go
@@ -3768,7 +3768,10 @@ func (c *Compile) newShuffleJoinScopeList(probeScopes, buildScopes []*Scope, n *
 		}
 	}
 
-	dop := plan2.GetShuffleDop(c.ncpu, len(cnlist), n.Stats.HashmapStats.HashmapSize)
+	dop := c.ncpu //for dedup join, limit the dop max to ncpu
+	if n.JoinType != plan.Node_DEDUP {
+		dop = plan2.GetShuffleDop(c.ncpu, len(cnlist), n.Stats.HashmapStats.HashmapSize)
+	}
 
 	bucketNum := len(cnlist) * dop
 	shuffleJoins := make([]*Scope, 0, bucketNum)

diff --git a/pkg/vm/engine/disttae/local_disttae_datasource.go b/pkg/vm/engine/disttae/local_disttae_datasource.go
@@ -831,6 +831,16 @@ func (ls *LocalDisttaeDataSource) applyWorkspaceFlushedS3Deletes(
 
 	s3FlushedDeletes := ls.table.getTxn().cn_flushed_s3_tombstone_object_stats_list
 
+	var tombstones []objectio.ObjectStats
+	s3FlushedDeletes.Range(func(key, value any) bool {
+		tombstones = append(tombstones, key.(objectio.ObjectStats))
+		return true
+	})
+
+	if len(tombstones) == 0 {
+		return
+	}
+
 	release := func() {}
 	if deletedRows == nil {
 		bm := objectio.GetReusableBitmap()
@@ -839,12 +849,6 @@ func (ls *LocalDisttaeDataSource) applyWorkspaceFlushedS3Deletes(
 	}
 	defer release()
 
-	var tombstones []objectio.ObjectStats
-	s3FlushedDeletes.Range(func(key, value any) bool {
-		tombstones = append(tombstones, key.(objectio.ObjectStats))
-		return true
-	})
-
 	curr := 0
 	getTombstone := func() (*objectio.ObjectStats, error) {
 		if curr >= len(tombstones) {

diff --git a/pkg/vm/engine/tae/blockio/utils.go b/pkg/vm/engine/tae/blockio/utils.go
@@ -108,18 +108,25 @@ func GetTombstonesByBlockId(
 		skipObjectCnt      int
 		totalBlkCnt        int
 	)
+
 	if tombstoneObjectCnt, skipObjectCnt, totalBlkCnt, err = CheckTombstoneFile(
 		ctx, blockId[:], getTombstoneFileFn, onBlockSelectedFn, fs,
 	); err != nil {
 		return
 	}
 
-	v2.TxnReaderEachBLKLoadedTombstoneHistogram.Observe(float64(loadedBlkCnt))
-	v2.TxnReaderScannedTotalTombstoneHistogram.Observe(float64(tombstoneObjectCnt))
+	if loadedBlkCnt > 0 {
+		v2.TxnReaderEachBLKLoadedTombstoneHistogram.Observe(float64(loadedBlkCnt))
+	}
+
 	if tombstoneObjectCnt > 0 {
+		v2.TxnReaderScannedTotalTombstoneHistogram.Observe(float64(tombstoneObjectCnt))
+	}
+
+	if tombstoneObjectCnt > 0 && skipObjectCnt > 0 {
 		v2.TxnReaderTombstoneZMSelectivityHistogram.Observe(float64(skipObjectCnt) / float64(tombstoneObjectCnt))
 	}
-	if totalBlkCnt > 0 {
+	if totalBlkCnt > 0 && loadedBlkCnt > 0 {
 		v2.TxnReaderTombstoneBLSelectivityHistogram.Observe(float64(loadedBlkCnt) / float64(totalBlkCnt))
 	}