From 9c3cc83a03797c22b33906854faa81380daa6cd5 Mon Sep 17 00:00:00 2001 From: prpeh Date: Thu, 6 Jun 2024 23:20:52 +0700 Subject: [PATCH 1/4] cmd/geth, ethdb/pebble: polish method naming and code comment --- cmd/geth/chaincmd.go | 4 ++-- cmd/geth/dbcmd.go | 8 ++++---- ethdb/pebble/pebble.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index d787f340a38b..6bdb598c7350 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -337,7 +337,7 @@ func importChain(ctx *cli.Context) error { fmt.Printf("Import done in %v.\n\n", time.Since(start)) // Output pre-compaction stats mostly to see the import trashing - showLeveldbStats(db) + showDBStats(db) // Print the memory statistics used by the importing mem := new(runtime.MemStats) @@ -360,7 +360,7 @@ func importChain(ctx *cli.Context) error { } fmt.Printf("Compaction done in %v.\n\n", time.Since(start)) - showLeveldbStats(db) + showDBStats(db) return importErr } diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 742eadd5f368..5d438e42e551 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -407,7 +407,7 @@ func checkStateContent(ctx *cli.Context) error { return nil } -func showLeveldbStats(db ethdb.KeyValueStater) { +func showDBStats(db ethdb.KeyValueStater) { if stats, err := db.Stat("leveldb.stats"); err != nil { log.Warn("Failed to read database stats", "error", err) } else { @@ -427,7 +427,7 @@ func dbStats(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() - showLeveldbStats(db) + showDBStats(db) return nil } @@ -439,7 +439,7 @@ func dbCompact(ctx *cli.Context) error { defer db.Close() log.Info("Stats before compaction") - showLeveldbStats(db) + showDBStats(db) log.Info("Triggering compaction") if err := db.Compact(nil, nil); err != nil { @@ -447,7 +447,7 @@ func dbCompact(ctx *cli.Context) error { return err } log.Info("Stats after compaction") - showLeveldbStats(db) + showDBStats(db) return nil } diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 0fac07c9604e..64b4d599b2ac 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -416,7 +416,7 @@ func upperBound(prefix []byte) (limit []byte) { } // Stat returns the internal metrics of Pebble in a text format. It's a developer -// method to read everything there is to read independent of Pebble version. +// method to read everything there is to read, independent of Pebble version. // // The property is unused in Pebble as there's only one thing to retrieve. func (d *Database) Stat(property string) (string, error) { From 99569fd2d4f8bbdaf44f306662a1190dc4c9c491 Mon Sep 17 00:00:00 2001 From: prpeh Date: Fri, 7 Jun 2024 10:39:37 +0700 Subject: [PATCH 2/4] implement db stat for pebble --- cmd/geth/dbcmd.go | 4 ++-- ethdb/pebble/pebble.go | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 5d438e42e551..fbfdb48fc3ae 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -408,12 +408,12 @@ func checkStateContent(ctx *cli.Context) error { } func showDBStats(db ethdb.KeyValueStater) { - if stats, err := db.Stat("leveldb.stats"); err != nil { + if stats, err := db.Stat("stats"); err != nil { log.Warn("Failed to read database stats", "error", err) } else { fmt.Println(stats) } - if ioStats, err := db.Stat("leveldb.iostats"); err != nil { + if ioStats, err := db.Stat("iostats"); err != nil { log.Warn("Failed to read database iostats", "error", err) } else { fmt.Println(ioStats) diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 64b4d599b2ac..48ebdbe0f733 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -417,10 +417,21 @@ func upperBound(prefix []byte) (limit []byte) { // Stat returns the internal metrics of Pebble in a text format. It's a developer // method to read everything there is to read, independent of Pebble version. -// -// The property is unused in Pebble as there's only one thing to retrieve. func (d *Database) Stat(property string) (string, error) { - return d.db.Metrics().String(), nil + metrics := d.db.Metrics() + if property == "iostats" { + total := metrics.Total() + return fmt.Sprintf("Read(MB):%.5f Write(MB):%.5f", + float64(total.BytesRead)/1048576.0, // 1024*1024 + float64(total.BytesFlushed+total.BytesCompacted)/1048576.0), nil + } + if property == "disk.size" { + return fmt.Sprintf("%d", metrics.Total().Size), nil + } + if property == "stats" { + return metrics.String(), nil + } + return "", fmt.Errorf("pebble stat property %s does not exists", property) } // Compact flattens the underlying data store for the given key range. In essence, From ea17320272691c88dce4eaf3cbbfd8cbc7ac339d Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Wed, 19 Jun 2024 10:13:07 +0800 Subject: [PATCH 3/4] cmd, core, ethdb, internal, trie: remove db property selector --- cmd/geth/dbcmd.go | 7 +---- core/rawdb/table.go | 4 +-- ethdb/database.go | 4 +-- ethdb/leveldb/leveldb.go | 54 +++++++++++++++++++++++++++++++------ ethdb/memorydb/memorydb.go | 4 +-- ethdb/pebble/pebble.go | 17 ++---------- ethdb/remotedb/remotedb.go | 4 +-- internal/ethapi/api.go | 4 +-- internal/web3ext/web3ext.go | 1 - trie/trie_test.go | 2 +- 10 files changed, 60 insertions(+), 41 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index fbfdb48fc3ae..f63aba07f155 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -408,16 +408,11 @@ func checkStateContent(ctx *cli.Context) error { } func showDBStats(db ethdb.KeyValueStater) { - if stats, err := db.Stat("stats"); err != nil { + if stats, err := db.Stat(); err != nil { log.Warn("Failed to read database stats", "error", err) } else { fmt.Println(stats) } - if ioStats, err := db.Stat("iostats"); err != nil { - log.Warn("Failed to read database iostats", "error", err) - } else { - fmt.Println(ioStats) - } } func dbStats(ctx *cli.Context) error { diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 19e4ed5b5c4b..4687ff07fef6 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -148,8 +148,8 @@ func (t *table) NewIterator(prefix []byte, start []byte) ethdb.Iterator { } // Stat returns a particular internal stat of the database. -func (t *table) Stat(property string) (string, error) { - return t.db.Stat(property) +func (t *table) Stat() (string, error) { + return t.db.Stat() } // Compact flattens the underlying data store for the given key range. In essence, diff --git a/ethdb/database.go b/ethdb/database.go index 3ec1f70e3b60..9ef5942a136b 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -39,8 +39,8 @@ type KeyValueWriter interface { // KeyValueStater wraps the Stat method of a backing data store. type KeyValueStater interface { - // Stat returns a particular internal stat of the database. - Stat(property string) (string, error) + // Stat returns the statistic data of the database. + Stat() (string, error) } // Compacter wraps the Compact method of a backing data store. diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 64f51cf21701..92838ad7ab85 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -22,7 +22,6 @@ package leveldb import ( "fmt" - "strings" "sync" "time" @@ -244,14 +243,53 @@ func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { return &snapshot{db: snap}, nil } -// Stat returns a particular internal stat of the database. -func (db *Database) Stat(property string) (string, error) { - if property == "" { - property = "leveldb.stats" - } else if !strings.HasPrefix(property, "leveldb.") { - property = "leveldb." + property +// Stat returns the statistic data of the database. +func (db *Database) Stat() (string, error) { + var stats leveldb.DBStats + if err := db.db.Stats(&stats); err != nil { + return "", err } - return db.db.GetProperty(property) + var ( + message string + totalRead int64 + totalWrite int64 + totalSize int64 + totalTables int + totalDuration time.Duration + ) + if len(stats.LevelSizes) > 0 { + message += " Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB)\n" + + "-------+------------+---------------+---------------+---------------+---------------\n" + for level, size := range stats.LevelSizes { + read := stats.LevelRead[level] + write := stats.LevelWrite[level] + duration := stats.LevelDurations[level] + tables := stats.LevelTablesCounts[level] + + if tables == 0 && duration == 0 { + continue + } + totalTables += tables + totalSize += size + totalRead += read + totalWrite += write + totalDuration += duration + message += fmt.Sprintf(" %3d | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n", + level, tables, float64(size)/1048576.0, duration.Seconds(), + float64(read)/1048576.0, float64(write)/1048576.0) + } + message += "-------+------------+---------------+---------------+---------------+---------------\n" + message += fmt.Sprintf(" Total | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n", + totalTables, float64(totalSize)/1048576.0, totalDuration.Seconds(), + float64(totalRead)/1048576.0, float64(totalWrite)/1048576.0) + message += "-------+------------+---------------+---------------+---------------+---------------\n\n" + } + message += fmt.Sprintf("Read(MB):%.5f Write(MB):%.5f\n", float64(stats.IORead)/1048576.0, float64(stats.IOWrite)/1048576.0) + message += fmt.Sprintf("BlockCache(MB):%.5f FileCache:%d\n", float64(stats.BlockCacheSize)/1048576.0, stats.OpenedTablesCount) + message += fmt.Sprintf("MemoryCompaction:%d Level0Compaction:%d NonLevel0Compaction:%d SeekCompaction:%d\n", stats.MemComp, stats.Level0Comp, stats.NonLevel0Comp, stats.SeekComp) + message += fmt.Sprintf("WriteDelayCount:%d WriteDelayDuration:%s Paused:%t\n", stats.WriteDelayCount, common.PrettyDuration(stats.WriteDelayDuration), stats.WritePaused) + message += fmt.Sprintf("Snapshots:%d Iterators:%d\n", stats.AliveSnapshots, stats.AliveIterators) + return message, nil } // Compact flattens the underlying data store for the given key range. In essence, diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index d1233acb2198..fa306c56d1b3 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -183,8 +183,8 @@ func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { } // Stat returns a particular internal stat of the database. -func (db *Database) Stat(property string) (string, error) { - return "", errors.New("unknown property") +func (db *Database) Stat() (string, error) { + return "", nil } // Compact is not supported on a memory database, but there's no need either as diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 48ebdbe0f733..130d6617b5ba 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -417,21 +417,8 @@ func upperBound(prefix []byte) (limit []byte) { // Stat returns the internal metrics of Pebble in a text format. It's a developer // method to read everything there is to read, independent of Pebble version. -func (d *Database) Stat(property string) (string, error) { - metrics := d.db.Metrics() - if property == "iostats" { - total := metrics.Total() - return fmt.Sprintf("Read(MB):%.5f Write(MB):%.5f", - float64(total.BytesRead)/1048576.0, // 1024*1024 - float64(total.BytesFlushed+total.BytesCompacted)/1048576.0), nil - } - if property == "disk.size" { - return fmt.Sprintf("%d", metrics.Total().Size), nil - } - if property == "stats" { - return metrics.String(), nil - } - return "", fmt.Errorf("pebble stat property %s does not exists", property) +func (d *Database) Stat() (string, error) { + return d.db.Metrics().String(), nil } // Compact flattens the underlying data store for the given key range. In essence, diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index c1c803caf2b9..bfecde4a3263 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -126,8 +126,8 @@ func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { panic("not supported") } -func (db *Database) Stat(property string) (string, error) { - panic("not supported") +func (db *Database) Stat() (string, error) { + return "", nil } func (db *Database) AncientDatadir() (string, error) { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index f211dcc6598b..242a79c1f1e0 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -2109,8 +2109,8 @@ func (api *DebugAPI) PrintBlock(ctx context.Context, number uint64) (string, err } // ChaindbProperty returns leveldb properties of the key-value database. -func (api *DebugAPI) ChaindbProperty(property string) (string, error) { - return api.b.ChainDb().Stat(property) +func (api *DebugAPI) ChaindbProperty() (string, error) { + return api.b.ChainDb().Stat() } // ChaindbCompact flattens the entire key-value database into a single level, diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 1da7d737dd94..4a1a37d722d6 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -263,7 +263,6 @@ web3._extend({ new web3._extend.Method({ name: 'chaindbProperty', call: 'debug_chaindbProperty', - params: 1, outputFormatter: console.log }), new web3._extend.Method({ diff --git a/trie/trie_test.go b/trie/trie_test.go index da60a7423dff..d2b63db142e1 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -820,7 +820,7 @@ func (s *spongeDb) Delete(key []byte) error { panic("implement func (s *spongeDb) NewBatch() ethdb.Batch { return &spongeBatch{s} } func (s *spongeDb) NewBatchWithSize(size int) ethdb.Batch { return &spongeBatch{s} } func (s *spongeDb) NewSnapshot() (ethdb.Snapshot, error) { panic("implement me") } -func (s *spongeDb) Stat(property string) (string, error) { panic("implement me") } +func (s *spongeDb) Stat() (string, error) { panic("implement me") } func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") } func (s *spongeDb) Close() error { return nil } func (s *spongeDb) Put(key []byte, value []byte) error { From 4a21582d203d07a911c9cb88ae0a13339347d102 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Wed, 19 Jun 2024 11:11:20 +0800 Subject: [PATCH 4/4] cmd, core, ethdb: fix function description --- cmd/geth/dbcmd.go | 7 ++++--- core/rawdb/table.go | 2 +- ethdb/memorydb/memorydb.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index f63aba07f155..4b683569a45e 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -408,11 +408,12 @@ func checkStateContent(ctx *cli.Context) error { } func showDBStats(db ethdb.KeyValueStater) { - if stats, err := db.Stat(); err != nil { + stats, err := db.Stat() + if err != nil { log.Warn("Failed to read database stats", "error", err) - } else { - fmt.Println(stats) + return } + fmt.Println(stats) } func dbStats(ctx *cli.Context) error { diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 4687ff07fef6..90849fe9d93e 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -147,7 +147,7 @@ func (t *table) NewIterator(prefix []byte, start []byte) ethdb.Iterator { } } -// Stat returns a particular internal stat of the database. +// Stat returns the statistic data of the database. func (t *table) Stat() (string, error) { return t.db.Stat() } diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index fa306c56d1b3..9b0872f89a33 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -182,7 +182,7 @@ func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { return newSnapshot(db), nil } -// Stat returns a particular internal stat of the database. +// Stat returns the statistic data of the database. func (db *Database) Stat() (string, error) { return "", nil }