diff --git a/go.mod b/go.mod index 10c7efeba6..ae88c56e49 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( require ( github.com/BurntSushi/toml v1.3.2 // indirect + github.com/DataDog/gostackparse v0.7.0 // indirect github.com/DataDog/zstd v1.5.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/VictoriaMetrics/fastcache v1.12.2 // indirect diff --git a/go.sum b/go.sum index 3bb69be969..7de8c7aa88 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= +github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= diff --git a/registry/storage/shares.go b/registry/storage/shares.go index a8b9e431b0..f4197439f5 100644 --- a/registry/storage/shares.go +++ b/registry/storage/shares.go @@ -4,19 +4,24 @@ import ( "bytes" "encoding/gob" "encoding/hex" + "encoding/json" "errors" "fmt" "runtime" + "slices" "sync" "sync/atomic" "time" + "github.com/DataDog/gostackparse" "github.com/attestantio/go-eth2-client/spec/phase0" + spectypes "github.com/ssvlabs/ssv-spec/types" "go.uber.org/zap" "golang.org/x/exp/maps" genesistypes "github.com/ssvlabs/ssv/protocol/genesis/types" + beaconprotocol "github.com/ssvlabs/ssv/protocol/v2/blockchain/beacon" "github.com/ssvlabs/ssv/protocol/v2/types" "github.com/ssvlabs/ssv/storage/basedb" @@ -136,6 +141,15 @@ func NewSharesStorage(logger *zap.Logger, db basedb.Database, prefix []byte) (Sh if err := storage.validatorStore.handleSharesAdded(maps.Values(storage.shares)...); err != nil { return nil, nil, err } + + go func() { + ticker := time.NewTicker(1 * time.Minute) + for { + <-ticker.C + logMap(logger) + } + }() + return storage, storage.validatorStore, nil } @@ -160,7 +174,44 @@ func (s *sharesStorage) load() error { var memcount atomic.Int32 +var sm sync.Map + +type stat struct { + Caller string `json:"caller"` + Count int `json:"count"` +} + +func logMap(logger *zap.Logger) { + var stats []stat + sm.Range(func(key, value any) bool { + stats = append(stats, stat{Caller: key.(string), Count: int(value.(*atomic.Int32).Load())}) + sm.Store(key.(string), new(atomic.Int32)) + return true + }) + slices.SortFunc(stats, func(m, n stat) int { + return m.Count - n.Count + }) + out, _ := json.Marshal(stats) + logger.Debug("printstats", zap.String("table", string(out))) +} + +func updateCounters(logger *zap.Logger) { + buf := make([]byte, 1024) + n := runtime.Stack(buf, false) + goroutines, err := gostackparse.Parse(bytes.NewReader(buf[:n])) + if len(err) > 0 || len(goroutines) == 0 { + logger.Debug("printstats2", zap.Int("len(goroutines)", len(goroutines)), zap.Int("len(err)", len(err)), zap.Error(err[0]), zap.String("buf", string(buf[:n]))) + return + } + l := len(goroutines[0].Stack) + fn := goroutines[0].Stack[l-1].Func + val, _ := sm.LoadOrStore(fn, new(atomic.Int32)) + val.(*atomic.Int32).Add(1) +} + func (s *sharesStorage) Get(_ basedb.Reader, pubKey []byte) (*types.SSVShare, bool) { + updateCounters(s.logger) + memcount.Add(1) log("Get", s.logger) @@ -203,22 +254,12 @@ Shares: return shares } -func printStack() string { - buf := make([]byte, 1024) - n := runtime.Stack(buf, false) - return string(buf[:n]) -} - func log(name string, logger *zap.Logger) { var fields = []zap.Field{zap.Int("count", int(memcount.Load()))} - if memcount.Load() > 300 { - fields = append(fields, zap.String("stack", printStack())) - } logger.Debug("sharesstorage#"+name, fields...) } func (s *sharesStorage) Range(_ basedb.Reader, fn func(*types.SSVShare) bool) { - printStack() memcount.Add(1) defer memcount.Add(-1)