From 500a9fbbbb7bcd999b8f768b79f8d29755449d8b Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Thu, 7 Feb 2019 13:29:32 +0100 Subject: [PATCH 1/5] swarm/storage: delete unused function parameters in tests 'chunksize' and 'processors' were unused; I did a cascading delete. --- swarm/storage/common_test.go | 16 +++---- swarm/storage/ldbstore_test.go | 78 ++++++++++++++-------------------- swarm/storage/memstore_test.go | 40 +++++++---------- 3 files changed, 55 insertions(+), 79 deletions(-) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 6c737af44ebf..75d2fed29f07 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -83,7 +83,7 @@ func newLDBStore(t *testing.T) (*LDBStore, func()) { return db, cleanup } -func mputRandomChunks(store ChunkStore, n int, chunksize int64) ([]Chunk, error) { +func mputRandomChunks(store ChunkStore, n int) ([]Chunk, error) { return mput(store, n, GenerateRandomChunk) } @@ -159,8 +159,8 @@ func (r *brokenLimitedReader) Read(buf []byte) (int, error) { return r.lr.Read(buf) } -func testStoreRandom(m ChunkStore, n int, chunksize int64, t *testing.T) { - chunks, err := mputRandomChunks(m, n, chunksize) +func testStoreRandom(m ChunkStore, n int, t *testing.T) { + chunks, err := mputRandomChunks(m, n) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -170,8 +170,8 @@ func testStoreRandom(m ChunkStore, n int, chunksize int64, t *testing.T) { } } -func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) { - chunks, err := mputRandomChunks(m, n, chunksize) +func testStoreCorrect(m ChunkStore, n int, t *testing.T) { + chunks, err := mputRandomChunks(m, n) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -195,7 +195,7 @@ func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) { } } -func benchmarkStorePut(store ChunkStore, n int, chunksize int64, b *testing.B) { +func benchmarkStorePut(store ChunkStore, n int, b *testing.B) { chunks := make([]Chunk, n) i := 0 f := func(dataSize int64) Chunk { @@ -222,8 +222,8 @@ func benchmarkStorePut(store ChunkStore, n int, chunksize int64, b *testing.B) { } } -func benchmarkStoreGet(store ChunkStore, n int, chunksize int64, b *testing.B) { - chunks, err := mputRandomChunks(store, n, chunksize) +func benchmarkStoreGet(store ChunkStore, n int, b *testing.B) { + chunks, err := mputRandomChunks(store, n) if err != nil { b.Fatalf("expected no error, got %v", err) } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index c87351323d67..9e7aba545d7b 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -78,22 +78,22 @@ func testPoFunc(k Address) (ret uint8) { return uint8(Proximity(basekey, k[:])) } -func testDbStoreRandom(n int, chunksize int64, mock bool, t *testing.T) { +func testDbStoreRandom(n int, mock bool, t *testing.T) { db, cleanup, err := newTestDbStore(mock, true) defer cleanup() if err != nil { t.Fatalf("init dbStore failed: %v", err) } - testStoreRandom(db, n, chunksize, t) + testStoreRandom(db, n, t) } -func testDbStoreCorrect(n int, chunksize int64, mock bool, t *testing.T) { +func testDbStoreCorrect(n int, mock bool, t *testing.T) { db, cleanup, err := newTestDbStore(mock, false) defer cleanup() if err != nil { t.Fatalf("init dbStore failed: %v", err) } - testStoreCorrect(db, n, chunksize, t) + testStoreCorrect(db, n, t) } func TestMarkAccessed(t *testing.T) { @@ -137,35 +137,35 @@ func TestMarkAccessed(t *testing.T) { } func TestDbStoreRandom_1(t *testing.T) { - testDbStoreRandom(1, 0, false, t) + testDbStoreRandom(1, false, t) } func TestDbStoreCorrect_1(t *testing.T) { - testDbStoreCorrect(1, 4096, false, t) + testDbStoreCorrect(1, false, t) } func TestDbStoreRandom_1k(t *testing.T) { - testDbStoreRandom(1000, 0, false, t) + testDbStoreRandom(1000, false, t) } func TestDbStoreCorrect_1k(t *testing.T) { - testDbStoreCorrect(1000, 4096, false, t) + testDbStoreCorrect(1000, false, t) } func TestMockDbStoreRandom_1(t *testing.T) { - testDbStoreRandom(1, 0, true, t) + testDbStoreRandom(1, true, t) } func TestMockDbStoreCorrect_1(t *testing.T) { - testDbStoreCorrect(1, 4096, true, t) + testDbStoreCorrect(1, true, t) } func TestMockDbStoreRandom_1k(t *testing.T) { - testDbStoreRandom(1000, 0, true, t) + testDbStoreRandom(1000, true, t) } func TestMockDbStoreCorrect_1k(t *testing.T) { - testDbStoreCorrect(1000, 4096, true, t) + testDbStoreCorrect(1000, true, t) } func testDbStoreNotFound(t *testing.T, mock bool) { @@ -242,54 +242,38 @@ func TestMockIterator(t *testing.T) { testIterator(t, true) } -func benchmarkDbStorePut(n int, processors int, chunksize int64, mock bool, b *testing.B) { +func benchmarkDbStorePut(n int, mock bool, b *testing.B) { db, cleanup, err := newTestDbStore(mock, true) defer cleanup() if err != nil { b.Fatalf("init dbStore failed: %v", err) } - benchmarkStorePut(db, n, chunksize, b) + benchmarkStorePut(db, n, b) } -func benchmarkDbStoreGet(n int, processors int, chunksize int64, mock bool, b *testing.B) { +func benchmarkDbStoreGet(n int, mock bool, b *testing.B) { db, cleanup, err := newTestDbStore(mock, true) defer cleanup() if err != nil { b.Fatalf("init dbStore failed: %v", err) } - benchmarkStoreGet(db, n, chunksize, b) + benchmarkStoreGet(db, n, b) } -func BenchmarkDbStorePut_1_500(b *testing.B) { - benchmarkDbStorePut(500, 1, 4096, false, b) +func BenchmarkDbStorePut_500(b *testing.B) { + benchmarkDbStorePut(500, false, b) } -func BenchmarkDbStorePut_8_500(b *testing.B) { - benchmarkDbStorePut(500, 8, 4096, false, b) +func BenchmarkDbStoreGet_500(b *testing.B) { + benchmarkDbStoreGet(500, false, b) } -func BenchmarkDbStoreGet_1_500(b *testing.B) { - benchmarkDbStoreGet(500, 1, 4096, false, b) +func BenchmarkMockDbStorePut_500(b *testing.B) { + benchmarkDbStorePut(500, true, b) } -func BenchmarkDbStoreGet_8_500(b *testing.B) { - benchmarkDbStoreGet(500, 8, 4096, false, b) -} - -func BenchmarkMockDbStorePut_1_500(b *testing.B) { - benchmarkDbStorePut(500, 1, 4096, true, b) -} - -func BenchmarkMockDbStorePut_8_500(b *testing.B) { - benchmarkDbStorePut(500, 8, 4096, true, b) -} - -func BenchmarkMockDbStoreGet_1_500(b *testing.B) { - benchmarkDbStoreGet(500, 1, 4096, true, b) -} - -func BenchmarkMockDbStoreGet_8_500(b *testing.B) { - benchmarkDbStoreGet(500, 8, 4096, true, b) +func BenchmarkMockDbStoreGet_500(b *testing.B) { + benchmarkDbStoreGet(500, true, b) } // TestLDBStoreWithoutCollectGarbage tests that we can put a number of random chunks in the LevelDB store, and @@ -302,7 +286,7 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + chunks, err := mputRandomChunks(ldb, n) if err != nil { t.Fatal(err.Error()) } @@ -382,7 +366,7 @@ func testLDBStoreCollectGarbage(t *testing.T) { putCount = roundTarget } remaining -= putCount - chunks, err := mputRandomChunks(ldb, putCount, int64(ch.DefaultSize)) + chunks, err := mputRandomChunks(ldb, putCount) if err != nil { t.Fatal(err.Error()) } @@ -429,7 +413,7 @@ func TestLDBStoreAddRemove(t *testing.T) { defer cleanup() n := 100 - chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + chunks, err := mputRandomChunks(ldb, n) if err != nil { t.Fatalf(err.Error()) } @@ -576,7 +560,7 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + chunks, err := mputRandomChunks(ldb, n) if err != nil { t.Fatal(err.Error()) } @@ -589,7 +573,7 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { t.Fatalf("fail add chunk #%d - %s: %v", i, chunks[i].Address(), err) } } - _, err = mputRandomChunks(ldb, 2, int64(ch.DefaultSize)) + _, err = mputRandomChunks(ldb, 2) if err != nil { t.Fatal(err.Error()) } @@ -621,7 +605,7 @@ func TestCleanIndex(t *testing.T) { ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks, err := mputRandomChunks(ldb, n, 4096) + chunks, err := mputRandomChunks(ldb, n) if err != nil { t.Fatal(err) } @@ -752,7 +736,7 @@ func TestCleanIndex(t *testing.T) { } // check that the iterator quits properly - chunks, err = mputRandomChunks(ldb, 4100, 4096) + chunks, err = mputRandomChunks(ldb, 4100) if err != nil { t.Fatal(err) } diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 6850d2d69a9d..8aaf486a7bf4 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -28,32 +28,32 @@ func newTestMemStore() *MemStore { return NewMemStore(storeparams, nil) } -func testMemStoreRandom(n int, chunksize int64, t *testing.T) { +func testMemStoreRandom(n int, t *testing.T) { m := newTestMemStore() defer m.Close() - testStoreRandom(m, n, chunksize, t) + testStoreRandom(m, n, t) } -func testMemStoreCorrect(n int, chunksize int64, t *testing.T) { +func testMemStoreCorrect(n int, t *testing.T) { m := newTestMemStore() defer m.Close() - testStoreCorrect(m, n, chunksize, t) + testStoreCorrect(m, n, t) } func TestMemStoreRandom_1(t *testing.T) { - testMemStoreRandom(1, 0, t) + testMemStoreRandom(1, t) } func TestMemStoreCorrect_1(t *testing.T) { - testMemStoreCorrect(1, 4104, t) + testMemStoreCorrect(1, t) } func TestMemStoreRandom_1k(t *testing.T) { - testMemStoreRandom(1000, 0, t) + testMemStoreRandom(1000, t) } func TestMemStoreCorrect_1k(t *testing.T) { - testMemStoreCorrect(100, 4096, t) + testMemStoreCorrect(100, t) } func TestMemStoreNotFound(t *testing.T) { @@ -66,32 +66,24 @@ func TestMemStoreNotFound(t *testing.T) { } } -func benchmarkMemStorePut(n int, processors int, chunksize int64, b *testing.B) { +func benchmarkMemStorePut(n int, b *testing.B) { m := newTestMemStore() defer m.Close() - benchmarkStorePut(m, n, chunksize, b) + benchmarkStorePut(m, n, b) } -func benchmarkMemStoreGet(n int, processors int, chunksize int64, b *testing.B) { +func benchmarkMemStoreGet(n int, b *testing.B) { m := newTestMemStore() defer m.Close() - benchmarkStoreGet(m, n, chunksize, b) + benchmarkStoreGet(m, n, b) } -func BenchmarkMemStorePut_1_500(b *testing.B) { - benchmarkMemStorePut(500, 1, 4096, b) +func BenchmarkMemStorePut_500(b *testing.B) { + benchmarkMemStorePut(500, b) } -func BenchmarkMemStorePut_8_500(b *testing.B) { - benchmarkMemStorePut(500, 8, 4096, b) -} - -func BenchmarkMemStoreGet_1_500(b *testing.B) { - benchmarkMemStoreGet(500, 1, 4096, b) -} - -func BenchmarkMemStoreGet_8_500(b *testing.B) { - benchmarkMemStoreGet(500, 8, 4096, b) +func BenchmarkMemStoreGet_500(b *testing.B) { + benchmarkMemStoreGet(500, b) } func TestMemStoreAndLDBStore(t *testing.T) { From 2102dec3f367fc5034b34511e67aada198823733 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Thu, 7 Feb 2019 13:34:42 +0100 Subject: [PATCH 2/5] swarm/storage: increase mput timeout for TestCleanIndex The following test failed with `-race` flag on Travis. --- FAIL: TestCleanIndex (43.85s) ldbstore_test.go:757: context deadline exceeded As `-race` can increase the run time by 10x. Alternative: we could just to increase the timeout for the specific test. As currently the timeout is hardcoded and (strangely) does not depend on the size of 'n'. That could be achieved by adding a new function mputWithCustomTimeout(), but then we should keep symmetry and add custom timeout for mget(), too. But I think the method already has too many parameters and might be replaced by the ongoing storage rewrite. So I won't do. --- swarm/storage/common_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 75d2fed29f07..52a28a888e77 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -91,7 +91,7 @@ func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error // put to localstore and wait for stored channel // does not check delivery error state errc := make(chan error) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() for i := int64(0); i < int64(n); i++ { chunk := f(ch.DefaultSize) From ac41a75499abf1ef0283e3f6d140dedc269fb832 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Thu, 7 Feb 2019 13:37:25 +0100 Subject: [PATCH 3/5] swarm/network: increase timeout for TestNetworkID The following test failed with `-race` flag on Travis. --- FAIL: TestNetworkID (9.04s) networkid_test.go:77: Error setting up network: context deadline exceeded As `-race` can increase the run time by 10x. --- swarm/network/networkid_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/network/networkid_test.go b/swarm/network/networkid_test.go index 8e68e5b52115..9d47cf9f6a35 100644 --- a/swarm/network/networkid_test.go +++ b/swarm/network/networkid_test.go @@ -44,7 +44,7 @@ var ( const ( NumberOfNets = 4 - MaxTimeout = 6 + MaxTimeout = 15 * time.Second ) func init() { @@ -146,7 +146,7 @@ func setupNetwork(numnodes int) (net *simulations.Network, err error) { return nil, fmt.Errorf("create node %d rpc client fail: %v", i, err) } //now setup and start event watching in order to know when we can upload - ctx, watchCancel := context.WithTimeout(context.Background(), MaxTimeout*time.Second) + ctx, watchCancel := context.WithTimeout(context.Background(), MaxTimeout) defer watchCancel() watchSubscriptionEvents(ctx, nodes[i].ID(), client, errc, quitC) //on every iteration we connect to all previous ones From baff204842952b58c9984a1489d7ce3c04ff5034 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Thu, 7 Feb 2019 13:44:06 +0100 Subject: [PATCH 4/5] swarm/network: decrease nodeCount for TestOverlaySim TestOverlaySim consistently failed with `-race` flag on Travis. My assumption was that we hit Travis' memory limitations. After decreasing the node count the test started to pass. Note: `-race` may increase memory usage by 5-10x === RUN TestOverlaySim panic: Could not startup node network for mocker goroutine 35 [running]: github.com/ethereum/go-ethereum/p2p/simulations.startStop(0xc0002a7000, 0xc0002a8b40, 0x10) /home/travis/gopath/src/github.com/ethereum/go-ethereum/p2p/simulations/mocker.go:66 +0x8a0 created by github.com/ethereum/go-ethereum/p2p/simulations.(*Server).StartMocker /home/travis/gopath/src/github.com/ethereum/go-ethereum/p2p/simulations/http.go:348 +0x260 FAIL github.com/ethereum/go-ethereum/swarm/network/simulations 2.788s --- swarm/network/simulations/overlay_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/network/simulations/overlay_test.go b/swarm/network/simulations/overlay_test.go index e56b64650353..636a5275918f 100644 --- a/swarm/network/simulations/overlay_test.go +++ b/swarm/network/simulations/overlay_test.go @@ -32,7 +32,7 @@ import ( ) var ( - nodeCount = 16 + nodeCount = 10 ) //This test is used to test the overlay simulation. From 64372e053939c4f798e1aa75afbc3048d63cc2c6 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Thu, 7 Feb 2019 14:14:43 +0100 Subject: [PATCH 5/5] swarm/network: increase context timeout for TestFileRetrieval The following test failed with `-race` flag on Travis. As `-race` might increase the execution time by 2-20x. --- FAIL: TestFileRetrieval (62.45s) snapshot_retrieval_test.go:61: context deadline exceeded FAIL github.com/ethereum/go-ethereum/swarm/network/stream 240.794s --- swarm/network/stream/snapshot_retrieval_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index f097e4180a6c..0f20f1be0606 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -151,7 +151,7 @@ func runFileRetrievalTest(nodeCount int) error { return err } - ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancelSimRun := context.WithTimeout(context.Background(), 3*time.Minute) defer cancelSimRun() result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {