From ee17de016a82c702f4326cfba91142e3ac9356a5 Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Sat, 28 Dec 2019 13:36:20 +0530 Subject: [PATCH 1/3] Fix windows build --- backup_test.go | 10 ++++++++-- db2_test.go | 35 ++++++++++++++++++++++++++--------- db_test.go | 3 +++ manifest_test.go | 1 + value.go | 2 +- value_test.go | 5 +++++ 6 files changed, 44 insertions(+), 12 deletions(-) diff --git a/backup_test.go b/backup_test.go index 16ef8ed5c..d2aba23d8 100644 --- a/backup_test.go +++ b/backup_test.go @@ -129,6 +129,7 @@ func TestBackupRestore2(t *testing.T) { if err != nil { t.Fatal(err) } + defer db1.Close() key1 := []byte("key1") key2 := []byte("key2") rawValue := []byte("NotLongValue") @@ -164,6 +165,7 @@ func TestBackupRestore2(t *testing.T) { if err != nil { t.Fatal(err) } + defer db2.Close() err = db2.Load(&backup, 16) if err != nil { t.Fatal(err) @@ -215,6 +217,7 @@ func TestBackupRestore2(t *testing.T) { if err != nil { t.Fatal(err) } + defer db3.Close() err = db3.Load(&backup, 16) if err != nil { @@ -330,7 +333,6 @@ func TestBackup(t *testing.T) { func TestBackupRestore3(t *testing.T) { var bb bytes.Buffer - tmpdir, err := ioutil.TempDir("", "badger-test") if err != nil { t.Fatal(err) @@ -346,7 +348,7 @@ func TestBackupRestore3(t *testing.T) { if err != nil { t.Fatal(err) } - + defer db1.Close() require.NoError(t, populateEntries(db1, entries)) _, err = db1.Backup(&bb, 0) @@ -361,6 +363,7 @@ func TestBackupRestore3(t *testing.T) { if err != nil { t.Fatal(err) } + defer db2.Close() require.NoError(t, db2.Load(&bb, 16)) // verify @@ -406,6 +409,7 @@ func TestBackupLoadIncremental(t *testing.T) { if err != nil { t.Fatal(err) } + defer db1.Close() require.NoError(t, populateEntries(db1, entries)) since, err := db1.Backup(&bb, 0) @@ -466,6 +470,8 @@ func TestBackupLoadIncremental(t *testing.T) { if err != nil { t.Fatal(err) } + defer db2.Close() + require.NoError(t, db2.Load(&bb, 16)) // verify diff --git a/db2_test.go b/db2_test.go index 3885f9aae..d9952af45 100644 --- a/db2_test.go +++ b/db2_test.go @@ -627,22 +627,29 @@ func TestL0GCBug(t *testing.T) { return []byte(fmt.Sprintf("%10d", i)) } val := []byte{1, 1, 1, 1, 1, 1, 1, 1} - // Insert 100 entries. This will create about 50 vlog files and 2 SST files. - for i := 0; i < 100; i++ { - err = db1.Update(func(txn *Txn) error { - return txn.SetEntry(NewEntry(key(i), val)) - }) - require.NoError(t, err) + // Insert 100 entries. This will create about 50*3 vlog files and 6 SST files. + for i := 0; i < 3; i++ { + for j := 0; j < 100; j++ { + err = db1.Update(func(txn *Txn) error { + return txn.SetEntry(NewEntry(key(j), val)) + }) + require.NoError(t, err) + } } // Run value log GC multiple times. This would ensure at least // one value log file is garbage collected. + success := 0 for i := 0; i < 10; i++ { err := db1.RunValueLogGC(0.01) + if err == nil { + success++ + } if err != nil && err != ErrNoRewrite { t.Fatalf(err.Error()) } } - + // Ensure alteast one GC call was successful. + require.NotZero(t, success) // CheckKeys reads all the keys previously stored. checkKeys := func(db *DB) { for i := 0; i < 100; i++ { @@ -665,7 +672,12 @@ func TestL0GCBug(t *testing.T) { if db1.valueDirGuard != nil { require.NoError(t, db1.valueDirGuard.release()) } - require.NoError(t, db1.vlog.Close()) + for _, f := range db1.vlog.filesMap { + require.NoError(t, f.fd.Close()) + } + require.NoError(t, db1.registry.Close()) + require.NoError(t, db1.lc.close()) + require.NoError(t, db1.manifest.close()) db2, err := Open(opts) require.NoError(t, err) @@ -723,7 +735,6 @@ func TestWindowsDataLoss(t *testing.T) { opt.Truncate = true db, err = Open(opt) require.NoError(t, err) - // Return after reading one entry. We're simulating a crash. // Simulate a crash by not closing db but releasing the locks. if db.dirLockGuard != nil { @@ -735,6 +746,12 @@ func TestWindowsDataLoss(t *testing.T) { // Don't use vlog.Close here. We don't want to fix the file size. Only un-mmap // the data so that we can truncate the file durning the next vlog.Open. require.NoError(t, y.Munmap(db.vlog.filesMap[db.vlog.maxFid].fmap)) + for _, f := range db.vlog.filesMap { + require.NoError(t, f.fd.Close()) + } + require.NoError(t, db.registry.Close()) + require.NoError(t, db.manifest.close()) + require.NoError(t, db.lc.close()) fmt.Println() fmt.Println("Third DB Open") diff --git a/db_test.go b/db_test.go index 82f4027df..9db02c92d 100644 --- a/db_test.go +++ b/db_test.go @@ -1161,6 +1161,9 @@ func TestExpiryImproperDBClose(t *testing.T) { // it would return Truncate Required Error. require.NoError(t, db0.vlog.Close()) + require.NoError(t, db0.registry.Close()) + require.NoError(t, db0.manifest.close()) + db1, err := Open(opt) require.NoError(t, err) err = db1.View(func(txn *Txn) error { diff --git a/manifest_test.go b/manifest_test.go index c9a51fc60..babc475aa 100644 --- a/manifest_test.go +++ b/manifest_test.go @@ -167,6 +167,7 @@ func TestOverlappingKeyRangeError(t *testing.T) { require.NoError(t, err) defer removeDir(dir) kv, err := Open(DefaultOptions(dir)) + defer kv.Close() require.NoError(t, err) lh0 := newLevelHandler(kv, 0) diff --git a/value.go b/value.go index cdf575a29..9ca97f245 100644 --- a/value.go +++ b/value.go @@ -197,7 +197,7 @@ func (lf *logFile) encryptionEnabled() bool { } func (lf *logFile) munmap() (err error) { - if lf.loadingMode != options.MemoryMap { + if lf.loadingMode != options.MemoryMap || len(lf.fmap) == 0 { // Nothing to do return nil } diff --git a/value_test.go b/value_test.go index 304ac7d41..8aa7c0909 100644 --- a/value_test.go +++ b/value_test.go @@ -780,6 +780,9 @@ func TestPenultimateLogCorruption(t *testing.T) { if db0.valueDirGuard != nil { require.NoError(t, db0.valueDirGuard.release()) } + require.NoError(t, db0.vlog.Close()) + require.NoError(t, db0.manifest.close()) + require.NoError(t, db0.registry.Close()) opt.Truncate = true db1, err := Open(opt) @@ -799,7 +802,9 @@ func TestPenultimateLogCorruption(t *testing.T) { func checkKeys(t *testing.T, kv *DB, keys [][]byte) { i := 0 txn := kv.NewTransaction(false) + defer txn.Discard() iter := txn.NewIterator(IteratorOptions{}) + defer iter.Close() for iter.Seek(keys[0]); iter.Valid(); iter.Next() { require.Equal(t, iter.Item().Key(), keys[i]) i++ From b2421416c1eb61498931ad43ed823f868e3667aa Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Sat, 28 Dec 2019 22:34:09 +0530 Subject: [PATCH 2/3] Fix deepsource warnings --- db_test.go | 18 +++++++++--------- manifest_test.go | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/db_test.go b/db_test.go index 9db02c92d..7d7e8dcd7 100644 --- a/db_test.go +++ b/db_test.go @@ -1677,7 +1677,7 @@ func ExampleOpen() { defer removeDir(dir) db, err := Open(DefaultOptions(dir)) if err != nil { - log.Fatal(err) + panic(err) } defer db.Close() @@ -1689,7 +1689,7 @@ func ExampleOpen() { }) if err != nil { - log.Fatal(err) + panic(err) } txn := db.NewTransaction(true) // Read-write txn @@ -1699,7 +1699,7 @@ func ExampleOpen() { } err = txn.Commit() if err != nil { - log.Fatal(err) + panic(err) } err = db.View(func(txn *Txn) error { @@ -1716,7 +1716,7 @@ func ExampleOpen() { }) if err != nil { - log.Fatal(err) + panic(err) } // Output: @@ -1727,13 +1727,13 @@ func ExampleOpen() { func ExampleTxn_NewIterator() { dir, err := ioutil.TempDir("", "badger-test") if err != nil { - log.Fatal(err) + panic(err) } defer removeDir(dir) db, err := Open(DefaultOptions(dir)) if err != nil { - log.Fatal(err) + panic(err) } defer db.Close() @@ -1751,13 +1751,13 @@ func ExampleTxn_NewIterator() { for i := 0; i < n; i++ { err := txn.SetEntry(NewEntry(bkey(i), bval(i))) if err != nil { - log.Fatal(err) + panic(err) } } err = txn.Commit() if err != nil { - log.Fatal(err) + panic(err) } opt := DefaultIteratorOptions @@ -1774,7 +1774,7 @@ func ExampleTxn_NewIterator() { return nil }) if err != nil { - log.Fatal(err) + panic(err) } fmt.Printf("Counted %d elements", count) // Output: diff --git a/manifest_test.go b/manifest_test.go index babc475aa..5062b3f1b 100644 --- a/manifest_test.go +++ b/manifest_test.go @@ -167,8 +167,8 @@ func TestOverlappingKeyRangeError(t *testing.T) { require.NoError(t, err) defer removeDir(dir) kv, err := Open(DefaultOptions(dir)) - defer kv.Close() require.NoError(t, err) + defer kv.Close() lh0 := newLevelHandler(kv, 0) lh1 := newLevelHandler(kv, 1) From 27b7e262d0b35fdb7f55421ee0a386ab0e343346 Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Tue, 7 Jan 2020 16:40:49 +0530 Subject: [PATCH 3/3] Replace log.fatal => panic to fix deepsource warnings --- db_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/db_test.go b/db_test.go index 7d7e8dcd7..98cdc1fa3 100644 --- a/db_test.go +++ b/db_test.go @@ -1203,7 +1203,7 @@ func randBytes(n int) []byte { recv := make([]byte, n) in, err := rand.Read(recv) if err != nil { - log.Fatal(err) + panic(err) } return recv[:in] } @@ -1672,7 +1672,7 @@ func TestGoroutineLeak(t *testing.T) { func ExampleOpen() { dir, err := ioutil.TempDir("", "badger-test") if err != nil { - log.Fatal(err) + panic(err) } defer removeDir(dir) db, err := Open(DefaultOptions(dir)) @@ -1695,7 +1695,7 @@ func ExampleOpen() { txn := db.NewTransaction(true) // Read-write txn err = txn.SetEntry(NewEntry([]byte("key"), []byte("value"))) if err != nil { - log.Fatal(err) + panic(err) } err = txn.Commit() if err != nil { @@ -1959,7 +1959,7 @@ func TestMain(m *testing.M) { // call flag.Parse() here if TestMain uses flags go func() { if err := http.ListenAndServe("localhost:8080", nil); err != nil { - log.Fatalf("Unable to open http port at 8080") + panic("Unable to open http port at 8080") } }() os.Exit(m.Run()) @@ -1979,12 +1979,12 @@ func ExampleDB_Subscribe() { // Open the DB. dir, err := ioutil.TempDir("", "badger-test") if err != nil { - log.Fatal(err) + panic(err) } defer removeDir(dir) db, err := Open(DefaultOptions(dir)) if err != nil { - log.Fatal(err) + panic(err) } defer db.Close() @@ -2003,7 +2003,7 @@ func ExampleDB_Subscribe() { } } if err := db.Subscribe(ctx, cb, prefix); err != nil && err != context.Canceled { - log.Fatal(err) + panic(err) } log.Printf("subscription closed") }() @@ -2013,11 +2013,11 @@ func ExampleDB_Subscribe() { // Write both keys, but only one should be printed in the Output. err = db.Update(func(txn *Txn) error { return txn.Set(aKey, aValue) }) if err != nil { - log.Fatal(err) + panic(err) } err = db.Update(func(txn *Txn) error { return txn.Set(bKey, bValue) }) if err != nil { - log.Fatal(err) + panic(err) } log.Printf("stopping subscription")