counter stats

go.mod

@@ -56,6 +56,7 @@ require (
 
 require (
 	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/DataDog/gostackparse v0.7.0 // indirect
 	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/VictoriaMetrics/fastcache v1.12.2 // indirect

go.sum

@@ -14,6 +14,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=
+github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
 github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
 github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=

registry/storage/shares.go

@@ -4,19 +4,24 @@ import (
 	"bytes"
 	"encoding/gob"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"runtime"
+	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/DataDog/gostackparse"
 	"github.com/attestantio/go-eth2-client/spec/phase0"
+
 	spectypes "github.com/ssvlabs/ssv-spec/types"
 	"go.uber.org/zap"
 	"golang.org/x/exp/maps"
 
 	genesistypes "github.com/ssvlabs/ssv/protocol/genesis/types"
+
 	beaconprotocol "github.com/ssvlabs/ssv/protocol/v2/blockchain/beacon"
 	"github.com/ssvlabs/ssv/protocol/v2/types"
 	"github.com/ssvlabs/ssv/storage/basedb"
@@ -136,6 +141,15 @@ func NewSharesStorage(logger *zap.Logger, db basedb.Database, prefix []byte) (Sh
 	if err := storage.validatorStore.handleSharesAdded(maps.Values(storage.shares)...); err != nil {
 		return nil, nil, err
 	}
+
+	go func() {
+		ticker := time.NewTicker(1 * time.Minute)
+		for {
+			<-ticker.C
+			logMap(logger)
+		}
+	}()
+
 	return storage, storage.validatorStore, nil
 }
 
@@ -160,7 +174,44 @@ func (s *sharesStorage) load() error {
 
 var memcount atomic.Int32
 
+var sm sync.Map
+
+type stat struct {
+	Caller string `json:"caller"`
+	Count  int    `json:"count"`
+}
+
+func logMap(logger *zap.Logger) {
+	var stats []stat
+	sm.Range(func(key, value any) bool {
+		stats = append(stats, stat{Caller: key.(string), Count: int(value.(*atomic.Int32).Load())})
+		sm.Store(key.(string), new(atomic.Int32))
+		return true
+	})
+	slices.SortFunc(stats, func(m, n stat) int {
+		return m.Count - n.Count
+	})
+	out, _ := json.Marshal(stats)
+	logger.Debug("printstats", zap.String("table", string(out)))
+}
+
+func updateCounters(logger *zap.Logger) {
+	buf := make([]byte, 1024)
+	n := runtime.Stack(buf, false)
+	goroutines, _ := gostackparse.Parse(bytes.NewReader(buf[:n]))
+	if len(goroutines) == 0 {
+		logger.Debug("printstats2", zap.String("buf", string(buf[:n])))
+		return
+	}
+	l := len(goroutines[0].Stack)
+	fn := goroutines[0].Stack[l-1].Func
+	val, _ := sm.LoadOrStore(fn, new(atomic.Int32))
+	val.(*atomic.Int32).Add(1)
+}
+
 func (s *sharesStorage) Get(_ basedb.Reader, pubKey []byte) (*types.SSVShare, bool) {
+	updateCounters(s.logger)
+
 	memcount.Add(1)
 	log("Get", s.logger)
 
@@ -203,22 +254,12 @@ Shares:
 	return shares
 }
 
-func printStack() string {
-	buf := make([]byte, 1024)
-	n := runtime.Stack(buf, false)
-	return string(buf[:n])
-}
-
 func log(name string, logger *zap.Logger) {
 	var fields = []zap.Field{zap.Int("count", int(memcount.Load()))}
-	if memcount.Load() > 300 {
-		fields = append(fields, zap.String("stack", printStack()))
-	}
 	logger.Debug("sharesstorage#"+name, fields...)
 }
 
 func (s *sharesStorage) Range(_ basedb.Reader, fn func(*types.SSVShare) bool) {
-	printStack()
 	memcount.Add(1)
 	defer memcount.Add(-1)