From 5412c27bce52c86c9b4b5411abda7851de353b09 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 2 Aug 2023 10:11:14 +0200 Subject: [PATCH] core/rawdb, ethdb/pebble: avoid fsync db in tests --- core/blockchain_repair_test.go | 3 +++ core/blockchain_sethead_test.go | 1 + core/blockchain_snapshot_test.go | 2 ++ core/rawdb/database.go | 7 +++++-- core/rawdb/databases_64bit.go | 4 ++-- ethdb/pebble/pebble.go | 15 +++++++++------ 6 files changed, 22 insertions(+), 10 deletions(-) diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 7080e23f8fe4..b2df39d17bb8 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -1767,6 +1767,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, AncientsDirectory: ancient, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) @@ -1847,6 +1848,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s db, err = rawdb.Open(rawdb.OpenOptions{ Directory: datadir, AncientsDirectory: ancient, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) @@ -1968,6 +1970,7 @@ func testIssue23496(t *testing.T, scheme string) { db, err = rawdb.Open(rawdb.OpenOptions{ Directory: datadir, AncientsDirectory: ancient, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 7691936ce241..fa739f924f5b 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -1971,6 +1971,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, AncientsDirectory: ancient, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index cdbd9d24fee1..dd012c430c4d 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -68,6 +68,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, AncientsDirectory: ancient, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) @@ -258,6 +259,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { newdb, err := rawdb.Open(rawdb.OpenOptions{ Directory: snaptest.datadir, AncientsDirectory: snaptest.ancient, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 981690f7bd3c..bc4ef113f95b 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -352,6 +352,9 @@ type OpenOptions struct { Cache int // the capacity(in megabytes) of the data caching Handles int // number of files to be open simultaneously ReadOnly bool + // Ephemeral means that filesystem sync operations should be avoided: data integrity in the face of + // a crash is not important. This option should typically be used in tests. + Ephemeral bool } // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. @@ -374,7 +377,7 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { if o.Type == dbPebble || existingDb == dbPebble { if PebbleEnabled { log.Info("Using pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) } else { return nil, errors.New("db.engine 'pebble' not supported on this platform") } @@ -387,7 +390,7 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { // on supported platforms and LevelDB on anything else. if PebbleEnabled { log.Info("Defaulting to pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) } else { log.Info("Defaulting to leveldb as the backing database") return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) diff --git a/core/rawdb/databases_64bit.go b/core/rawdb/databases_64bit.go index 73bfeb20838f..1593e89bfe6a 100644 --- a/core/rawdb/databases_64bit.go +++ b/core/rawdb/databases_64bit.go @@ -28,8 +28,8 @@ const PebbleEnabled = true // NewPebbleDBDatabase creates a persistent key-value database without a freezer // moving immutable chain segments into cold storage. -func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { - db, err := pebble.New(file, cache, handles, namespace, readonly) +func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) { + db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral) if err != nil { return nil, err } diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index a06f59bcfae4..12a84cc91a6e 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -84,6 +84,8 @@ type Database struct { writeDelayStartTime time.Time // The start time of the latest write stall writeDelayCount atomic.Int64 // Total number of write stall counts writeDelayTime atomic.Int64 // Total time spent in write stalls + + writeOptions *pebble.WriteOptions } func (d *Database) onCompactionBegin(info pebble.CompactionInfo) { @@ -118,7 +120,7 @@ func (d *Database) onWriteStallEnd() { // New returns a wrapped pebble DB object. The namespace is the prefix that the // metrics reporting should use for surfacing internal stats. -func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) { +func New(file string, cache int, handles int, namespace string, readonly bool, ephemeral bool) (*Database, error) { // Ensure we have some minimal caching and file guarantees if cache < minCache { cache = minCache @@ -142,9 +144,10 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( memTableSize = maxMemTableSize } db := &Database{ - fn: file, - log: logger, - quitChan: make(chan chan error), + fn: file, + log: logger, + quitChan: make(chan chan error), + writeOptions: &pebble.WriteOptions{Sync: !ephemeral}, } opt := &pebble.Options{ // Pebble has a single combined cache area and the write @@ -279,7 +282,7 @@ func (d *Database) Put(key []byte, value []byte) error { if d.closed { return pebble.ErrClosed } - return d.db.Set(key, value, pebble.Sync) + return d.db.Set(key, value, d.writeOptions) } // Delete removes the key from the key-value store. @@ -535,7 +538,7 @@ func (b *batch) Write() error { if b.db.closed { return pebble.ErrClosed } - return b.b.Commit(pebble.Sync) + return b.b.Commit(b.db.writeOptions) } // Reset resets the batch for reuse.