Skip to content

Commit

Permalink
Wit consumer (#2660)
Browse files Browse the repository at this point in the history 
* add wit protocol handler (basic)

* add serializers

* wit configs

* protocol not disconnecting wit

* CachingStore with tests

* witness collector with tests

* witnessing store with tests

* injecting witness collector to block processor

* witness db but witness not yet served

* test witness persistence

* witness caching

* syn server tests

* serializer tests

* DB fix and handlign empty witnesses more efficiently

* witnessing store used

* fx witness collector init

* storage witnesses

* witness logs

* fixed witness db in producer scenarios

* added test

* null annotations and fixed tests in NUllWitness cases

* fix cli test

* fixed ethereum tests

* one more fix for ethereum tests

* fixing NDM build

* more parallelized tests (and shorter)

* wit protocol handler tests

* improved code after suggestions from Lukasz

* diag messages

* fixed the message code

* sending witnesses

* fixed seq rlp

* added regression test

* test for trinity msg deserialization

* message id space fix for eth65

* restore 17 message space

* trinity head 0

* add implied status message

* add a comment

* probably insignificant change

* test fix

* regression test for message space

* add wit protocol handler (basic)

* add serializers

* wit configs

* protocol not disconnecting wit

* CachingStore with tests

* witness collector with tests

* witnessing store with tests

* injecting witness collector to block processor

* witness db but witness not yet served

* test witness persistence

* witness caching

* syn server tests

* serializer tests

* DB fix and handlign empty witnesses more efficiently

* witnessing store used

* fx witness collector init

* storage witnesses

* witness logs

* fixed witness db in producer scenarios

* added test

* null annotations and fixed tests in NUllWitness cases

* fix cli test

* fixed ethereum tests

* one more fix for ethereum tests

* fixing NDM build

* more parallelized tests (and shorter)

* wit protocol handler tests

* improved code after suggestions from Lukasz

* diag messages

* fixed the message code

* sending witnesses

* fixed seq rlp

* added regression test

* test for trinity msg deserialization

* message id space fix for eth65

* restore 17 message space

* trinity head 0

* add implied status message

* add a comment

* probably insignificant change

* test fix

* regression test for message space

* fix build

* build fixes

* build and sln fixes

* sln fix

* null session in test ok

* quick fix for wit

* diagnostic message

* DbFactory init

* Remove baseline dbs from core code

* Init DbFactory

* IPlugableDbConfig experiment

* Work on BaselineDbConfig

* Refactor DbOnTheRocks constructors

* db naming fix

* Registering from db providers

* fix db path issue

* Removing NDM database dependencies from core

* Fix MemDbProvider

* Add previously ignored files

* cosmetic fixes

* Fix RocksDbProvider

* Cosmetic fix v2

* Add BaselineDbProviderTests.cs

* Add NdmDbProviderTests

* Add RpcProviderTests

* Add config tests

* Init Db factories

* New way for defining plugin dbs

* Move ndm metrics, refactor plugins db providers

* further refactoring

* Work on StandardDbInitializer

* Add StandardDbInitializer

* Refactor RocksDbProvider

* Revert launchSettings.json

* Remove redundant classes

* Columns Dbs

* Refactor ReadOnlyDbProvider

* Refactored BeamProvider

* Fix tests

* Fix tests, temp comments

* ndm build fix

* Fix JSONRpc tests

* Fix readonly interface

* Fix Runner tests

* Add Metrics

* Fix baseline db initialization

* build fix

* Fix RegisterReadOnlyDb

* Remove PluggableDbConfig & Fix tests

* Refactor DbInitializer & refactor NdmConsumer

* Fix TracerModule tests & remove rest of plugins dbs dependencies from configs

* StandardDbInitializer refactoring

* Fix initializers

* Add NdmDbInitializer Tests

* ConsumerDb tests

* Fix initialzer

* DbProvider ignore cases

* Add some db provider tests

* Fix failing test

* Cosmetic fixes

* Review fixes

* Fix pendingtxs

* Add StandardDbInitializer tests

* build fixes

* fix merge

* fix the beam sync database

* Add BlockTree.NewSuggestedBlock

* SyncConfig.WitnessProtocolEnabled

* Witness sync

* Add satellite protocols and wintess dowanloading

* Fix collecting witness

* Move state sync limits to StateSyncFeed.MaxRequestSize

Move WitnessBlockSyncFeed to AllocationContexts.Witness
Move WitnessBlockSyncFeed to SortedSet based on: retries, blocknumber

* fixes + tests

* add BlockTree.NewSuggestedBlock tests

* protocol refactor

* Touch code in witness

* Add witness pruner

* fix Node formating

* Add MaxBlocksInQueue on WitnessBlockSyncFeed retries

* Add WitnessPruner log

* Be more liberal with WitnessStateSyncFeed

* Lru cache fix

* don't disconnect on exceptions for static nodes

* Improve WitnessBlockSyncFeed retries and Allocation strategies

* Improve static connectivity

* WitnessStateSyncFeed max 10 retries

* post merge fixes

* some fixes and better cod readablity

* post merge fix

Co-authored-by: MarekM25 <marekm2504@gmail.com>
Co-authored-by: lukasz.rozmej <lukasz.rozmej@gmail.com>
  • Loading branch information
@tkstanczak @MarekM25 @LukaszRozmej
3 people authored Jan 16, 2021
1 parent 9d1b7cd commit d372dda
Show file tree
Hide file tree
Showing 212 changed files with 4,192 additions and 788 deletions.
173 changes: 173 additions & 0 deletions src/Nethermind/Chains/wit.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
{
"name": "CliqueTrinity",
"dataDir": "clique_trinity",
"engine": {
"clique": {
"params": {
"period": 15,
"epoch": 30000
}
}
},
"params": {
"accountStartNonce": "0x0",
"chainID": "0x5",
"eip140Transition": "0x0",
"eip145Transition": "0x0",
"eip150Transition": "0x0",
"eip155Transition": "0x0",
"eip160Transition": "0x0",
"eip161abcTransition": "0x0",
"eip161dTransition": "0x0",
"eip211Transition": "0x0",
"eip214Transition": "0x0",
"eip658Transition": "0x0",
"eip1014Transition": "0x0",
"eip1052Transition": "0x0",
"eip1283Transition": "0x0",
"eip1283DisableTransition": "0x0",
"eip152Transition": "0x17D433",
"eip1108Transition": "0x17D433",
"eip1344Transition": "0x17D433",
"eip1884Transition": "0x17D433",
"eip2028Transition": "0x17D433",
"eip2200Transition": "0x17D433",
"gasLimitBoundDivisor": "0x400",
"maxCodeSize": "0x6000",
"maxCodeSizeTransition": "0x0",
"maximumExtraDataSize": "0xffff",
"minGasLimit": "0x1388",
"networkID": "0x44"
},
"genesis": {
"seal": {
"ethereum": {
"nonce": "0x0000000000000000",
"mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000"
}
},
"author": "0x0000000000000000000000000000000000000000",
"difficulty": "0x1",
"extraData": "0x22466c6578692069732061207468696e6722202d204166726900000000000000ff7c66068fb536b0a463ad82436b8baf741617290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
"gasLimit": "0xa00000",
"parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"timestamp": "0x5bdda800"
},
"nodes": [
"enode://cedb232376896fd6753f4ed4c805cd5492aaadc484aa53626ac481e80b65cedea635c5ac9b2cd9db6e724d9fe40f2cdf86639591bf6dbc931e08dfd70276537d@94.237.80.80:31505"
],
"accounts": {
"0x0000000000000000000000000000000000000000": {
"balance": "0x1"
},
"0x0000000000000000000000000000000000000001": {
"balance": "0x1",
"builtin": {
"name": "ecrecover",
"pricing": {
"linear": {
"base": 3000,
"word": 0
}
}
}
},
"0x0000000000000000000000000000000000000002": {
"balance": "0x1",
"builtin": {
"name": "sha256",
"pricing": {
"linear": {
"base": 60,
"word": 12
}
}
}
},
"0x0000000000000000000000000000000000000003": {
"balance": "0x1",
"builtin": {
"name": "ripemd160",
"pricing": {
"linear": {
"base": 600,
"word": 120
}
}
}
},
"0x0000000000000000000000000000000000000004": {
"balance": "0x1",
"builtin": {
"name": "identity",
"pricing": {
"linear": {
"base": 15,
"word": 3
}
}
}
},
"0x0000000000000000000000000000000000000005": {
"balance": "0x1",
"builtin": {
"name": "modexp",
"activate_at": "0x0",
"pricing": {
"modexp": {
"divisor": 20
}
}
}
},
"0x0000000000000000000000000000000000000006": {
"balance": "0x1",
"builtin": {
"name": "alt_bn128_add",
"activate_at": "0x0",
"pricing": {
"linear": {
"base": 500,
"word": 0
}
}
}
},
"0x0000000000000000000000000000000000000007": {
"balance": "0x1",
"builtin": {
"name": "alt_bn128_mul",
"activate_at": "0x0",
"pricing": {
"linear": {
"base": 40000,
"word": 0
}
}
}
},
"0x0000000000000000000000000000000000000008": {
"balance": "0x1",
"builtin": {
"name": "alt_bn128_pairing",
"activate_at": "0x0",
"pricing": {
"alt_bn128_pairing": {
"base": 100000,
"pair": 80000
}
}
}
},
"7e5f4552091a69125d5dfcb7b8c2659029395bdf": { "balance": "100000000000000000000" },
"2b5ad5c4795c026514f8317c7a215e218dccd6cf": { "balance": "100000000000000000000" },
"6813eb9362372eef6200f3b1dbc3f819671cba69": { "balance": "100000000000000000000" },
"1eff47bc3a10a45d4b230b5d10e37751fe6aa718": { "balance": "100000000000000000000" },
"e1ab8145f7e55dc933d51a18c793f901a3a0b276": { "balance": "100000000000000000000" },
"e57bfe9f44b819898f47bf37e5af72a0783e1141": { "balance": "100000000000000000000" },
"d41c057fd1c78805aac12b0a94a405c0461a6fbb": { "balance": "100000000000000000000" },
"f1f6619b38a98d6de0800f1defc0a6399eb6d30c": { "balance": "100000000000000000000" },
"f7edc8fa1ecc32967f827c9043fcae6ba73afa5c": { "balance": "100000000000000000000" },
"ff7c66068fb536b0a463ad82436b8baf74161729": { "balance": "100000000000000000000" }
}
}
1 change: 1 addition & 0 deletions src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch
storageProvider,
transactionPool,
receiptStorage,
NullWitnessCollector.Instance,
_logManager);

IBlockchainProcessor blockchainProcessor = new BlockchainProcessor(
Expand Down
3 changes: 3 additions & 0 deletions src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using Nethermind.Blockchain.Validators;
using Nethermind.Config;
using Nethermind.Consensus;
using Nethermind.Core;
using Nethermind.Evm;
using Nethermind.Facade;
using Nethermind.State;
Expand Down Expand Up @@ -58,6 +59,7 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory
/// DO NOT USE OUTSIDE OF PROCESSING BLOCK CONTEXT!
/// </remarks>
IStateProvider? StateProvider { get; set; }
IKeyValueStoreWithBatching? MainStateDbWithCache { get; set; }
IReadOnlyStateProvider? ChainHeadStateProvider { get; set; }
IStateReader? StateReader { get; set; }
IStorageProvider? StorageProvider { get; set; }
Expand All @@ -66,5 +68,6 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory
ITxSender? TxSender { get; set; }
ITxPool? TxPool { get; set; }
ITxPoolInfoProvider? TxPoolInfoProvider { get; set; }
IWitnessCollector? WitnessCollector { get; set; }
}
}
2 changes: 1 addition & 1 deletion src/Nethermind/Nethermind.Api/Nethermind.Api.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<AssemblyOriginatorKeyFile>strong.snk</AssemblyOriginatorKeyFile>
<Nullable>enable</Nullable>
<Nullable>annotations</Nullable>
</PropertyGroup>

<ItemGroup>
Expand Down
6 changes: 3 additions & 3 deletions src/Nethermind/Nethermind.AuRa.Test/Contract/TxPriorityContractTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ public async Task whitelist_should_return_correctly_with_local_storage([Values(t
{
if (chain.LocalDataSource.Data == null)
{
Assert.Fail("Local file rule storage wasn't loaded.");
Assert.Fail("Local file rule storage has not been loaded.");
}
}

Expand Down Expand Up @@ -206,7 +206,7 @@ public async Task priority_should_return_correctly_with_local_storage([Values(tr
{
if (chain.LocalDataSource.Data == null)
{
Assert.Fail("Local file rule storage wasn't loaded.");
Assert.Fail("Local file rule storage has not been loaded.");
}
}

Expand Down Expand Up @@ -252,7 +252,7 @@ public async Task mingas_should_return_correctly_with_local_storage([Values(true
{
if (chain.LocalDataSource.Data == null)
{
Assert.Fail("Local file rule storage wasn't loaded.");
Assert.Fail("Local file rule storage has not been loaded.");
}
}

Expand Down
35 changes: 35 additions & 0 deletions src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ public void Prepared_block_contains_author_field()
new StorageProvider(trieStore, stateProvider, LimboLogs.Instance),
NullTxPool.Instance,
NullReceiptStorage.Instance,
NullWitnessCollector.Instance,
LimboLogs.Instance);

BlockHeader header = Build.A.BlockHeader.WithAuthor(TestItem.AddressD).TestObject;
Expand All @@ -76,6 +77,39 @@ public void Prepared_block_contains_author_field()
Assert.AreEqual(1, processedBlocks.Length, "length");
Assert.AreEqual(block.Author, processedBlocks[0].Author, "author");
}

[Test]
public void Can_store_a_witness()
{
ISnapshotableDb stateDb = new StateDb();
ISnapshotableDb codeDb = new StateDb();
var trieStore = new TrieStore(stateDb, LimboLogs.Instance);

IStateProvider stateProvider = new StateProvider(trieStore, codeDb, LimboLogs.Instance);
ITransactionProcessor transactionProcessor = Substitute.For<ITransactionProcessor>();
IWitnessCollector witnessCollector = Substitute.For<IWitnessCollector>();
BlockProcessor processor = new BlockProcessor(
RinkebySpecProvider.Instance,
TestBlockValidator.AlwaysValid,
NoBlockRewards.Instance,
transactionProcessor,
stateProvider,
new StorageProvider(trieStore, stateProvider, LimboLogs.Instance),
NullTxPool.Instance,
NullReceiptStorage.Instance,
witnessCollector,
LimboLogs.Instance);

BlockHeader header = Build.A.BlockHeader.WithAuthor(TestItem.AddressD).TestObject;
Block block = Build.A.Block.WithHeader(header).TestObject;
_ = processor.Process(
Keccak.EmptyTreeHash,
new List<Block> {block},
ProcessingOptions.None,
NullBlockTracer.Instance);

witnessCollector.Received(1).Persist(block.Hash);
}

[Test]
public void Recovers_state_on_cancel()
Expand All @@ -94,6 +128,7 @@ public void Recovers_state_on_cancel()
new StorageProvider(trieStore, stateProvider, LimboLogs.Instance),
NullTxPool.Instance,
NullReceiptStorage.Instance,
NullWitnessCollector.Instance,
LimboLogs.Instance);

BlockHeader header = Build.A.BlockHeader.WithNumber(1).WithAuthor(TestItem.AddressD).TestObject;
Expand Down
23 changes: 23 additions & 0 deletions src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,17 @@ public void Add_genesis_shall_notify()
bool hasNotified = false;
BlockTree blockTree = BuildBlockTree();
blockTree.NewHeadBlock += (sender, args) => { hasNotified = true; };

bool hasNotifiedNewSuggested = false;
blockTree.NewSuggestedBlock += (sender, args) => { hasNotifiedNewSuggested = true; };

Block block = Build.A.Block.WithNumber(0).TestObject;
var result = blockTree.SuggestBlock(block);
blockTree.UpdateMainChain(block);

Assert.True(hasNotified, "notification");
Assert.AreEqual(AddBlockResult.Added, result, "result");
Assert.True(hasNotifiedNewSuggested, "NewSuggestedBlock");
}

[Test]
Expand All @@ -85,12 +89,16 @@ public void Add_genesis_shall_work_even_with_0_difficulty()
bool hasNotified = false;
BlockTree blockTree = BuildBlockTree();
blockTree.NewBestSuggestedBlock += (sender, args) => { hasNotified = true; };

bool hasNotifiedNewSuggested = false;
blockTree.NewSuggestedBlock += (sender, args) => { hasNotifiedNewSuggested = true; };

Block block = Build.A.Block.WithNumber(0).WithDifficulty(0).TestObject;
var result = blockTree.SuggestBlock(block);

Assert.True(hasNotified, "notification");
Assert.AreEqual(AddBlockResult.Added, result, "result");
Assert.True(hasNotifiedNewSuggested, "NewSuggestedBlock");
}

[Test]
Expand All @@ -112,11 +120,16 @@ public void Shall_notify_on_new_head_block_after_genesis()
Block block1 = Build.A.Block.WithNumber(1).WithDifficulty(2).WithParent(block0).TestObject;
blockTree.SuggestBlock(block0);
blockTree.NewHeadBlock += (sender, args) => { hasNotified = true; };

bool hasNotifiedNewSuggested = false;
blockTree.NewSuggestedBlock += (sender, args) => { hasNotifiedNewSuggested = true; };

var result = blockTree.SuggestBlock(block1);
blockTree.UpdateMainChain(block1);

Assert.True(hasNotified, "notification");
Assert.AreEqual(AddBlockResult.Added, result, "result");
Assert.True(hasNotifiedNewSuggested, "NewSuggestedBlock");
}

[Test]
Expand All @@ -128,10 +141,15 @@ public void Shall_notify_on_new_suggested_block_after_genesis()
Block block1 = Build.A.Block.WithNumber(1).WithDifficulty(2).WithParent(block0).TestObject;
blockTree.SuggestBlock(block0);
blockTree.NewBestSuggestedBlock += (sender, args) => { hasNotified = true; };

bool hasNotifiedNewSuggested = false;
blockTree.NewSuggestedBlock += (sender, args) => { hasNotifiedNewSuggested = true; };

var result = blockTree.SuggestBlock(block1);

Assert.True(hasNotified, "notification");
Assert.AreEqual(AddBlockResult.Added, result, "result");
Assert.True(hasNotifiedNewSuggested, "NewSuggestedBlock");
}

[Test]
Expand All @@ -147,11 +165,16 @@ public void Shall_not_notify_but_add_on_lower_difficulty()
blockTree.SuggestBlock(block1);
blockTree.NewHeadBlock += (sender, args) => { hasNotifiedHead = true; };
blockTree.NewBestSuggestedBlock += (sender, args) => { hasNotifiedBest = true; };

bool hasNotifiedNewSuggested = false;
blockTree.NewSuggestedBlock += (sender, args) => { hasNotifiedNewSuggested = true; };

var result = blockTree.SuggestBlock(block2);

Assert.False(hasNotifiedBest, "notification best");
Assert.False(hasNotifiedHead, "notification head");
Assert.AreEqual(AddBlockResult.Added, result, "result");
Assert.True(hasNotifiedNewSuggested, "NewSuggestedBlock");
}

[Test]
Expand Down
2 changes: 2 additions & 0 deletions src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
using Nethermind.Specs;
using Nethermind.State;
using Nethermind.State.Repositories;
using Nethermind.State.Witnesses;
using Nethermind.Trie.Pruning;
using Nethermind.TxPool;
using Nethermind.TxPool.Storages;
Expand Down Expand Up @@ -94,6 +95,7 @@ public void Setup()
storageProvider,
txPool,
NullReceiptStorage.Instance,
new WitnessCollector(memDbProvider.StateDb, LimboLogs.Instance),
LimboLogs.Instance);
_blockchainProcessor = new BlockchainProcessor(
_blockTree,
Expand Down
Loading

0 comments on commit d372dda

Please sign in to comment.