From 4a1428d38dbcb255ab50f12c1d1ee2620c4394a9 Mon Sep 17 00:00:00 2001 From: dangershony Date: Wed, 8 Apr 2020 00:27:37 +0100 Subject: [PATCH 1/6] Add block header store --- .gitignore | 1 + .../CoinViewTests.cs | 4 +- .../Base/ChainRepositoryTest.cs | 4 +- src/Blockcore/Base/BaseFeature.cs | 1 + src/Blockcore/Base/ChainRepository.cs | 25 ++--- src/Blockcore/Blockcore.csproj | 1 + src/Blockcore/Configuration/DataFolder.cs | 3 + src/Blockcore/Consensus/LeveldbHeaderStore.cs | 91 +++++++++++++++++++ src/NBitcoin/ChainedHeader.cs | 74 ++++++++++----- src/NBitcoin/MemoryHeaderStore.cs | 40 ++++++++ 10 files changed, 207 insertions(+), 37 deletions(-) create mode 100644 src/Blockcore/Consensus/LeveldbHeaderStore.cs create mode 100644 src/NBitcoin/MemoryHeaderStore.cs diff --git a/.gitignore b/.gitignore index 82a697c2c..aff23dbfe 100644 --- a/.gitignore +++ b/.gitignore @@ -263,3 +263,4 @@ src/.idea/.idea.Stratis.Bitcoin.FullNode/.idea/ .DS_Store *.iml +/src/.idea diff --git a/src/Blockcore.IntegrationTests/CoinViewTests.cs b/src/Blockcore.IntegrationTests/CoinViewTests.cs index eb12cc9b1..35d7c4af4 100644 --- a/src/Blockcore.IntegrationTests/CoinViewTests.cs +++ b/src/Blockcore.IntegrationTests/CoinViewTests.cs @@ -279,7 +279,7 @@ private ChainedHeader MakeNext(ChainedHeader previous, Network network) [Fact] public void CanSaveChainIncrementally() { - using (var repo = new ChainRepository(TestBase.CreateTestDir(this), this.loggerFactory, this.dBreezeSerializer)) + using (var repo = new ChainRepository(TestBase.CreateTestDir(this), this.loggerFactory, this.dBreezeSerializer, new MemoryHeaderStore())) { var chain = new ChainIndexer(this.regTest); @@ -322,4 +322,4 @@ private ChainedHeader AppendBlock(params ChainIndexer[] chainsIndexer) return this.AppendBlock(index, chainsIndexer); } } -} +} \ No newline at end of file diff --git a/src/Blockcore.Tests/Base/ChainRepositoryTest.cs b/src/Blockcore.Tests/Base/ChainRepositoryTest.cs index 2e76e0f30..1a133db10 100644 --- a/src/Blockcore.Tests/Base/ChainRepositoryTest.cs +++ b/src/Blockcore.Tests/Base/ChainRepositoryTest.cs @@ -27,7 +27,7 @@ public void SaveWritesChainToDisk() var chain = new ChainIndexer(KnownNetworks.StratisRegTest); this.AppendBlock(chain); - using (var repo = new ChainRepository(dir, new LoggerFactory(), this.dBreezeSerializer)) + using (var repo = new ChainRepository(dir, new LoggerFactory(), this.dBreezeSerializer, new MemoryHeaderStore())) { repo.SaveAsync(chain).GetAwaiter().GetResult(); } @@ -74,7 +74,7 @@ public void GetChainReturnsConcurrentChainFromDisk() transaction.Commit(); } } - using (var repo = new ChainRepository(dir, new LoggerFactory(), this.dBreezeSerializer)) + using (var repo = new ChainRepository(dir, new LoggerFactory(), this.dBreezeSerializer, new MemoryHeaderStore())) { var testChain = new ChainIndexer(KnownNetworks.StratisRegTest); testChain.SetTip(repo.LoadAsync(testChain.Genesis).GetAwaiter().GetResult()); diff --git a/src/Blockcore/Base/BaseFeature.cs b/src/Blockcore/Base/BaseFeature.cs index 853d04ad1..97fae4816 100644 --- a/src/Blockcore/Base/BaseFeature.cs +++ b/src/Blockcore/Base/BaseFeature.cs @@ -389,6 +389,7 @@ public static IFullNodeBuilder UseBaseFeature(this IFullNodeBuilder fullNodeBuil services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Blockcore/Base/ChainRepository.cs b/src/Blockcore/Base/ChainRepository.cs index 69f8be217..ab1163526 100644 --- a/src/Blockcore/Base/ChainRepository.cs +++ b/src/Blockcore/Base/ChainRepository.cs @@ -25,6 +25,7 @@ public interface IChainRepository : IDisposable public class ChainRepository : IChainRepository { private readonly DBreezeSerializer dBreezeSerializer; + private readonly IBlockHeaderStore blockHeaderStore; /// Instance logger. private readonly ILogger logger; @@ -34,9 +35,10 @@ public class ChainRepository : IChainRepository private BlockLocator locator; - public ChainRepository(string folder, ILoggerFactory loggerFactory, DBreezeSerializer dBreezeSerializer) + public ChainRepository(string folder, ILoggerFactory loggerFactory, DBreezeSerializer dBreezeSerializer, IBlockHeaderStore blockHeaderStore) { this.dBreezeSerializer = dBreezeSerializer; + this.blockHeaderStore = blockHeaderStore; Guard.NotEmpty(folder, nameof(folder)); Guard.NotNull(loggerFactory, nameof(loggerFactory)); @@ -46,8 +48,8 @@ public ChainRepository(string folder, ILoggerFactory loggerFactory, DBreezeSeria this.dbreeze = new DBreezeEngine(folder); } - public ChainRepository(DataFolder dataFolder, ILoggerFactory loggerFactory, DBreezeSerializer dBreezeSerializer) - : this(dataFolder.ChainPath, loggerFactory, dBreezeSerializer) + public ChainRepository(DataFolder dataFolder, ILoggerFactory loggerFactory, DBreezeSerializer dBreezeSerializer, IBlockHeaderStore blockHeaderStore) + : this(dataFolder.ChainPath, loggerFactory, dBreezeSerializer, blockHeaderStore) { } @@ -65,21 +67,22 @@ public Task LoadAsync(ChainedHeader genesisHeader) if (!firstRow.Exists) return genesisHeader; - BlockHeader previousHeader = this.dBreezeSerializer.Deserialize(firstRow.Value); - Guard.Assert(previousHeader.GetHash() == genesisHeader.HashBlock); // can't swap networks + BlockHeader nextHeader = this.dBreezeSerializer.Deserialize(firstRow.Value); + Guard.Assert(nextHeader.GetHash() == genesisHeader.HashBlock); // can't swap networks foreach (Row row in transaction.SelectForwardSkip("Chain", 1)) { - if ((tip != null) && (previousHeader.HashPrevBlock != tip.HashBlock)) + if ((tip != null) && (nextHeader.HashPrevBlock != tip.HashBlock)) break; BlockHeader blockHeader = this.dBreezeSerializer.Deserialize(row.Value); - tip = new ChainedHeader(previousHeader, blockHeader.HashPrevBlock, tip); - previousHeader = blockHeader; + tip = new ChainedHeader(nextHeader, blockHeader.HashPrevBlock, tip); + if (tip.Height == 0) tip.SetBlockHeaderStore(this.blockHeaderStore); + nextHeader = blockHeader; } - if (previousHeader != null) - tip = new ChainedHeader(previousHeader, previousHeader.GetHash(), tip); + if (nextHeader != null) + tip = new ChainedHeader(nextHeader, nextHeader.GetHash(), tip); if (tip == null) tip = genesisHeader; @@ -148,4 +151,4 @@ public void Dispose() this.dbreeze?.Dispose(); } } -} +} \ No newline at end of file diff --git a/src/Blockcore/Blockcore.csproj b/src/Blockcore/Blockcore.csproj index 4cd81d59b..042f7d606 100644 --- a/src/Blockcore/Blockcore.csproj +++ b/src/Blockcore/Blockcore.csproj @@ -52,6 +52,7 @@ + diff --git a/src/Blockcore/Configuration/DataFolder.cs b/src/Blockcore/Configuration/DataFolder.cs index f8c83374b..d2d273b39 100644 --- a/src/Blockcore/Configuration/DataFolder.cs +++ b/src/Blockcore/Configuration/DataFolder.cs @@ -25,6 +25,7 @@ public DataFolder(string path) this.CoindbPath = Path.Combine(path, "coindb"); this.AddressManagerFilePath = path; this.ChainPath = Path.Combine(path, "chain"); + this.HeadersPath = Path.Combine(path, "headers"); this.KeyValueRepositoryPath = Path.Combine(path, "common"); this.BlockPath = Path.Combine(path, "blocks"); this.PollsPath = Path.Combine(path, "polls"); @@ -55,6 +56,8 @@ public DataFolder(string path) /// public string ChainPath { get; internal set; } + public string HeadersPath { get; internal set; } + /// Path to the folder with separated key-value items managed by . public string KeyValueRepositoryPath { get; internal set; } diff --git a/src/Blockcore/Consensus/LeveldbHeaderStore.cs b/src/Blockcore/Consensus/LeveldbHeaderStore.cs new file mode 100644 index 000000000..1a1505805 --- /dev/null +++ b/src/Blockcore/Consensus/LeveldbHeaderStore.cs @@ -0,0 +1,91 @@ +using System; +using Blockcore.Configuration; +using Blockcore.Utilities; +using LevelDB; + +namespace NBitcoin +{ + public class LeveldbHeaderStore : IBlockHeaderStore + { + private readonly Network network; + + /// + /// Headers that are close to the tip + /// + private readonly MemoryCountCache headers; + + private readonly DB leveldb; + + private object locker; + + public LeveldbHeaderStore(Network network, DataFolder dataFolder, ChainIndexer chainIndexer) + { + this.network = network; + this.ChainIndexer = chainIndexer; + // this.headers = new Dictionary(); + this.headers = new MemoryCountCache(501); + this.locker = new object(); + + // Open a connection to a new DB and create if not found + var options = new Options { CreateIfMissing = true }; + this.leveldb = new DB(options, dataFolder.HeadersPath); + } + + public ChainIndexer ChainIndexer { get; } + + public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) + { + if (this.headers.TryGetValue(hash, out BlockHeader blockHeader)) + { + return blockHeader; + } + + byte[] bytes = hash.ToBytes(); + + lock (this.locker) + { + bytes = this.leveldb.Get(bytes); + } + + if (bytes == null) + { + throw new ApplicationException("Header must exist if requested"); + } + + blockHeader = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); + blockHeader.FromBytes(bytes, this.network.Consensus.ConsensusFactory); + + // If the header is 500 blocks behind tip or one block ahead cache it. + if ((chainedHeader.Height > this.ChainIndexer.Height - 500) && (chainedHeader.Height <= this.ChainIndexer.Height + 1)) + this.headers.AddOrUpdate(hash, blockHeader); + + return blockHeader; + } + + public bool StoreHeader(BlockHeader blockHeader) + { + ConsensusFactory consensusFactory = this.network.Consensus.ConsensusFactory; + + if (blockHeader is ProvenBlockHeader) + { + // If ProvenBlockHeader copy the header parameters. + BlockHeader newHeader = consensusFactory.CreateBlockHeader(); + newHeader.Bits = blockHeader.Bits; + newHeader.Time = blockHeader.Time; + newHeader.Nonce = blockHeader.Nonce; + newHeader.Version = blockHeader.Version; + newHeader.HashMerkleRoot = blockHeader.HashMerkleRoot; + newHeader.HashPrevBlock = blockHeader.HashPrevBlock; + + blockHeader = newHeader; + } + + lock (this.locker) + { + this.leveldb.Put(blockHeader.GetHash().ToBytes(), blockHeader.ToBytes(consensusFactory)); + } + + return true; + } + } +} \ No newline at end of file diff --git a/src/NBitcoin/ChainedHeader.cs b/src/NBitcoin/ChainedHeader.cs index cfadbfd5c..1c5fc45a3 100644 --- a/src/NBitcoin/ChainedHeader.cs +++ b/src/NBitcoin/ChainedHeader.cs @@ -75,7 +75,15 @@ public class ChainedHeader public int Height { get; private set; } /// Block header for this entry. - public BlockHeader Header { get; private set; } + //public BlockHeader Header { get; private set; } + + public BlockHeader Header + { + get + { + return this.HeaderStore.GetHeader(this, this.HashBlock); + } + } /// Integer representation of the . private BigInteger chainWork; @@ -89,6 +97,8 @@ public class ChainedHeader /// public ValidationState BlockValidationState { get; set; } + public IBlockHeaderStore HeaderStore { get; private set; } + /// /// An indicator that the current instance of has been disconnected from the previous instance. /// @@ -112,22 +122,24 @@ public bool IsReferenceConnected public List Next { get; private set; } /// - /// Constructs a chained block. + /// Set a different header store to the default , this can be done only on genesis header (height 0). /// - /// Header for the block. - /// Hash of the header of the block. - /// Link to the previous block in the chain. + public void SetBlockHeaderStore(IBlockHeaderStore blockHeaderStore) + { + if (this.Height != 0) + { + throw new ArgumentException("IBlockHeaderStore can only be set on the genesis header."); + } + + blockHeaderStore.StoreHeader(this.HeaderStore.GetHeader(this, this.HashBlock)); + this.HeaderStore = blockHeaderStore; + } + public ChainedHeader(BlockHeader header, uint256 headerHash, ChainedHeader previous) : this(header, headerHash) { if (previous != null) this.Height = previous.Height + 1; - if (this.Height == 0) - { - this.BlockDataAvailability = BlockDataAvailabilityState.BlockAvailable; - this.BlockValidationState = ValidationState.FullyValidated; - } - this.Previous = previous; if (previous == null) @@ -144,25 +156,42 @@ public ChainedHeader(BlockHeader header, uint256 headerHash, ChainedHeader previ this.Skip = this.Previous.GetAncestor(this.GetSkipHeight(this.Height)); } + if (this.Height == 0) + { + this.BlockDataAvailability = BlockDataAvailabilityState.BlockAvailable; + this.BlockValidationState = ValidationState.FullyValidated; + + this.HeaderStore = new MemoryHeaderStore(); + this.HeaderStore.StoreHeader(header); + } + else + { + this.HeaderStore = this.Previous.HeaderStore; + this.HeaderStore.StoreHeader(header); + } + this.CalculateChainWork(); } - /// - /// Constructs a chained header at the start of a chain. - /// - /// The header for the block. - /// The hash computed according to NetworkOptions. - /// The height of the block. public ChainedHeader(BlockHeader header, uint256 headerHash, int height) : this(header, headerHash) { this.Height = height; - this.CalculateChainWork(); if (height == 0) { this.BlockDataAvailability = BlockDataAvailabilityState.BlockAvailable; this.BlockValidationState = ValidationState.FullyValidated; + + this.HeaderStore = new MemoryHeaderStore(); + this.HeaderStore.StoreHeader(header); } + else + { + this.HeaderStore = this.Previous.HeaderStore; + this.HeaderStore.StoreHeader(header); + } + + this.CalculateChainWork(); } /// @@ -172,7 +201,7 @@ public ChainedHeader(BlockHeader header, uint256 headerHash, int height) : this( /// The hash of the block's header. private ChainedHeader(BlockHeader header, uint256 headerHash) { - this.Header = header ?? throw new ArgumentNullException(nameof(header)); + // this.Header = header ?? throw new ArgumentNullException(nameof(header)); this.HashBlock = headerHash ?? throw new ArgumentNullException(nameof(headerHash)); this.Next = new List(1); } @@ -275,7 +304,7 @@ public IEnumerable EnumerateToGenesis() /// public override string ToString() { - return this.Height + "-" + this.HashBlock + "-" + this.BlockValidationState + (this.Header is ProvenBlockHeader ? " - PH" : string.Empty); + return this.Height + "-" + this.HashBlock + "-" + this.BlockValidationState + (this.Header is ProvenBlockHeader ? " - PH" : string.Empty); } /// @@ -661,7 +690,8 @@ private int InvertLowestOne(int n) /// Use this method very carefully because it could cause race conditions if used at the wrong moment. public void SetHeader(BlockHeader newHeader) { - this.Header = newHeader; + //this.Header = newHeader; + this.HeaderStore.StoreHeader(newHeader); } } -} +} \ No newline at end of file diff --git a/src/NBitcoin/MemoryHeaderStore.cs b/src/NBitcoin/MemoryHeaderStore.cs new file mode 100644 index 000000000..42c98b2d2 --- /dev/null +++ b/src/NBitcoin/MemoryHeaderStore.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using NBitcoin.BouncyCastle.Math; +using NBitcoin.Crypto; + +namespace NBitcoin +{ + public interface IBlockHeaderStore + { + BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash); + + bool StoreHeader(BlockHeader blockHeader); + } + + public class MemoryHeaderStore : IBlockHeaderStore + { + private readonly ConcurrentDictionary headers; + + public MemoryHeaderStore() + { + this.headers = new ConcurrentDictionary(); + } + + public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) + { + if (!this.headers.TryGetValue(hash, out BlockHeader header)) + { + throw new ApplicationException("Header must exist if requested"); + } + + return header; + } + + public bool StoreHeader(BlockHeader blockHeader) + { + return this.headers.TryAdd(blockHeader.GetHash(), blockHeader); + } + } +} \ No newline at end of file From 733019c40d89dee87e21e9116cc5d18f79adb361 Mon Sep 17 00:00:00 2001 From: dangershony Date: Thu, 9 Apr 2020 17:16:53 +0100 Subject: [PATCH 2/6] Fix consensus puller bug assertion --- src/Blockcore/Consensus/ConsensusManager.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Blockcore/Consensus/ConsensusManager.cs b/src/Blockcore/Consensus/ConsensusManager.cs index 9e487a9f3..85f783862 100644 --- a/src/Blockcore/Consensus/ConsensusManager.cs +++ b/src/Blockcore/Consensus/ConsensusManager.cs @@ -1343,6 +1343,12 @@ private void ProcessDownloadQueueLocked() this.logger.LogDebug("With {0} average block size, we have {1} download slots available.", avgSize, maxBlocksToAsk); + if (maxBlocksToAsk <= 0) + { + this.logger.LogTrace("(-)[NOT_ENOUGH_FREE_BYTES]"); + return; + } + BlockDownloadRequest request = this.toDownloadQueue.Peek(); if (request.BlocksToDownload.Count <= maxBlocksToAsk) From ae28676166af455af49b1bbd82f85a0b3d1549d1 Mon Sep 17 00:00:00 2001 From: dangershony Date: Thu, 9 Apr 2020 23:55:31 +0100 Subject: [PATCH 3/6] Optimize chain work field as byte array insted of BigInteger --- .../Controllers/BlockStoreController.cs | 2 +- .../Controllers/FullNodeController.cs | 2 +- src/Blockcore/Consensus/ConsensusManager.cs | 2 +- src/NBitcoin/ChainedHeader.cs | 19 +++- src/NBitcoin/Target.cs | 93 +++++++++++-------- src/NBitcoin/UInt256.cs | 7 +- 6 files changed, 75 insertions(+), 50 deletions(-) diff --git a/src/Blockcore.Features.BlockStore/Controllers/BlockStoreController.cs b/src/Blockcore.Features.BlockStore/Controllers/BlockStoreController.cs index 8c65e174c..23a750914 100644 --- a/src/Blockcore.Features.BlockStore/Controllers/BlockStoreController.cs +++ b/src/Blockcore.Features.BlockStore/Controllers/BlockStoreController.cs @@ -129,7 +129,7 @@ public IActionResult GetBlock([FromQuery] SearchByHashRequest query) var posBlock = block as PosBlock; blockModel.PosBlockSignature = posBlock.BlockSignature.ToHex(this.network); - blockModel.PosBlockTrust = new Target(chainedHeader.GetBlockProof()).ToUInt256().ToString(); + blockModel.PosBlockTrust = new Target(chainedHeader.GetBlockTarget()).ToUInt256().ToString(); blockModel.PosChainTrust = chainedHeader.ChainWork.ToString(); // this should be similar to ChainWork } diff --git a/src/Blockcore.Features.RPC/Controllers/FullNodeController.cs b/src/Blockcore.Features.RPC/Controllers/FullNodeController.cs index 40f85978d..c71951fac 100644 --- a/src/Blockcore.Features.RPC/Controllers/FullNodeController.cs +++ b/src/Blockcore.Features.RPC/Controllers/FullNodeController.cs @@ -405,7 +405,7 @@ public object GetBlock(string blockHash, int verbosity = 1) var posBlock = block as PosBlock; blockModel.PosBlockSignature = posBlock.BlockSignature.ToHex(this.Network); - blockModel.PosBlockTrust = new Target(chainedHeader.GetBlockProof()).ToUInt256().ToString(); + blockModel.PosBlockTrust = new Target(chainedHeader.GetBlockTarget()).ToUInt256().ToString(); blockModel.PosChainTrust = chainedHeader.ChainWork.ToString(); // this should be similar to ChainWork if (this.stakeChain != null) diff --git a/src/Blockcore/Consensus/ConsensusManager.cs b/src/Blockcore/Consensus/ConsensusManager.cs index 85f783862..1bb9c3f8c 100644 --- a/src/Blockcore/Consensus/ConsensusManager.cs +++ b/src/Blockcore/Consensus/ConsensusManager.cs @@ -804,7 +804,7 @@ private async Task ConnectChainAsync(ListInteger representation of the . - private BigInteger chainWork; + /// The chain work field is represented as a byte array to reduce the memory foot print of a BigInteger + private byte[] chainWork; /// Total amount of work in the chain up to and including this block. - public uint256 ChainWork { get { return Target.ToUInt256(this.chainWork); } } + public uint256 ChainWork + { + get + { + return Target.ToUInt256(this.chainWork); + } + } /// public BlockDataAvailabilityState BlockDataAvailability { get; set; } @@ -201,7 +208,7 @@ public ChainedHeader(BlockHeader header, uint256 headerHash, int height) : this( /// The hash of the block's header. private ChainedHeader(BlockHeader header, uint256 headerHash) { - // this.Header = header ?? throw new ArgumentNullException(nameof(header)); + if (header == null) throw new ArgumentNullException(nameof(header)); this.HashBlock = headerHash ?? throw new ArgumentNullException(nameof(headerHash)); this.Next = new List(1); } @@ -211,14 +218,16 @@ private ChainedHeader(BlockHeader header, uint256 headerHash) /// private void CalculateChainWork() { - this.chainWork = (this.Previous == null ? BigInteger.Zero : this.Previous.chainWork).Add(this.GetBlockProof()); + BigInteger previousWork = this.Previous == null ? BigInteger.Zero : new BigInteger(this.Previous.chainWork); + this.chainWork = previousWork.Add(this.GetBlockTarget()).ToByteArray(); } /// Calculates the amount of work that this block contributes to the total chain work. /// Amount of work. - public BigInteger GetBlockProof() + public BigInteger GetBlockTarget() { BigInteger target = this.Header.Bits.ToBigInteger(); + if ((target.CompareTo(BigInteger.Zero) <= 0) || (target.CompareTo(Pow256) >= 0)) return BigInteger.Zero; diff --git a/src/NBitcoin/Target.cs b/src/NBitcoin/Target.cs index 20fc6b228..5f5e1beca 100644 --- a/src/NBitcoin/Target.cs +++ b/src/NBitcoin/Target.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Globalization; using System.Linq; using NBitcoin.BouncyCastle.Math; @@ -10,19 +11,18 @@ namespace NBitcoin /// public class Target { - private static Target _Difficulty1 = new Target(new byte[] { 0x1d, 0x00, 0xff, 0xff }); + private static Target difficulty1 = new Target(new byte[] { 0x1d, 0x00, 0xff, 0xff }); + public static Target Difficulty1 { get { - return _Difficulty1; + return difficulty1; } } - public Target(uint compact) - : this(ToBytes(compact)) + public Target(uint compact) : this(ToBytes(compact)) { - } private static byte[] ToBytes(uint bits) @@ -36,98 +36,102 @@ private static byte[] ToBytes(uint bits) }; } - - private BigInteger _Target; + private BigInteger target; public Target(byte[] compact) { - if(compact.Length == 4) + if (compact.Length == 4) { byte exp = compact[0]; var val = new BigInteger(compact.SafeSubarray(1, 3)); - this._Target = val.ShiftLeft(8 * (exp - 3)); + this.target = val.ShiftLeft(8 * (exp - 3)); } else + { throw new FormatException("Invalid number of bytes"); - } - + } + } + public Target(BigInteger target) { - this._Target = target; - this._Target = new Target(ToCompact())._Target; + this.target = target; + this.target = new Target(ToCompact()).target; } + public Target(uint256 target) { - this._Target = new BigInteger(target.ToBytes(false)); - this._Target = new Target(ToCompact())._Target; + this.target = new BigInteger(target.ToBytes(false)); + this.target = new Target(ToCompact()).target; } public static implicit operator Target(uint a) { return new Target(a); } + public static implicit operator uint(Target a) { - byte[] bytes = a._Target.ToByteArray(); + byte[] bytes = a.target.ToByteArray(); byte[] val = bytes.SafeSubarray(0, Math.Min(bytes.Length, 3)); Array.Reverse(val); byte exp = (byte)(bytes.Length); if (exp == 1 && bytes[0] == 0) exp = 0; int missing = 4 - val.Length; - if(missing > 0) + if (missing > 0) val = val.Concat(new byte[missing]).ToArray(); - if(missing < 0) + if (missing < 0) val = val.Take(-missing).ToArray(); return (uint)val[0] + (uint)(val[1] << 8) + (uint)(val[2] << 16) + (uint)(exp << 24); } - private double? _Difficulty; - + private double? difficulty; + public double Difficulty { get { - if(this._Difficulty == null) + if (this.difficulty == null) { - BigInteger[] qr = Difficulty1._Target.DivideAndRemainder(this._Target); + BigInteger[] qr = Difficulty1.target.DivideAndRemainder(this.target); BigInteger quotient = qr[0]; BigInteger remainder = qr[1]; BigInteger decimalPart = BigInteger.Zero; - for(int i = 0; i < 12; i++) + for (int i = 0; i < 12; i++) { - BigInteger div = (remainder.Multiply(BigInteger.Ten)).Divide(this._Target); + BigInteger div = (remainder.Multiply(BigInteger.Ten)).Divide(this.target); decimalPart = decimalPart.Multiply(BigInteger.Ten); decimalPart = decimalPart.Add(div); - remainder = remainder.Multiply(BigInteger.Ten).Subtract(div.Multiply(this._Target)); + remainder = remainder.Multiply(BigInteger.Ten).Subtract(div.Multiply(this.target)); } - this._Difficulty = double.Parse(quotient.ToString() + "." + decimalPart.ToString(), new NumberFormatInfo() + this.difficulty = double.Parse(quotient.ToString() + "." + decimalPart.ToString(), new NumberFormatInfo() { NegativeSign = "-", NumberDecimalSeparator = "." }); } - return this._Difficulty.Value; + return this.difficulty.Value; } } public override bool Equals(object obj) { var item = obj as Target; - if(item == null) + if (item == null) return false; - return this._Target.Equals(item._Target); + return this.target.Equals(item.target); } + public static bool operator ==(Target a, Target b) { - if(ReferenceEquals(a, b)) + if (ReferenceEquals(a, b)) return true; - if(((object)a == null) || ((object)b == null)) + if (((object)a == null) || ((object)b == null)) return false; - return a._Target.Equals(b._Target); + return a.target.Equals(b.target); } public static bool operator !=(Target a, Target b) @@ -137,12 +141,12 @@ public override bool Equals(object obj) public override int GetHashCode() { - return this._Target.GetHashCode(); + return this.target.GetHashCode(); } - + public BigInteger ToBigInteger() { - return this._Target; + return this.target; } public uint ToCompact() @@ -152,20 +156,27 @@ public uint ToCompact() public uint256 ToUInt256() { - return ToUInt256(this._Target); + return ToUInt256(this.target); } internal static uint256 ToUInt256(BigInteger input) { - byte[] array = input.ToByteArray(); + return ToUInt256(input.ToByteArray()); + } + internal static uint256 ToUInt256(byte[] array) + { int missingZero = 32 - array.Length; - if(missingZero < 0) + if (missingZero < 0) throw new InvalidOperationException("Awful bug, this should never happen"); - if(missingZero != 0) - return new uint256(new byte[missingZero].Concat(array), false); + if (missingZero != 0) + { + Span buffer = stackalloc byte[32]; + array.AsSpan().CopyTo(buffer.Slice(missingZero)); + return new uint256(buffer, false); + } return new uint256(array, false); } @@ -175,4 +186,4 @@ public override string ToString() return this.ToUInt256().ToString(); } } -} +} \ No newline at end of file diff --git a/src/NBitcoin/UInt256.cs b/src/NBitcoin/UInt256.cs index cc140a151..18f952710 100644 --- a/src/NBitcoin/UInt256.cs +++ b/src/NBitcoin/UInt256.cs @@ -91,7 +91,7 @@ public uint256(uint256 b) this.part4 = b.part4; } - public uint256(ReadOnlySpan input) + public uint256(ReadOnlySpan input, bool littleEndian = true) { if (input.Length != ExpectedSize) { @@ -101,6 +101,11 @@ public uint256(ReadOnlySpan input) Span dst = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref this.part1, ExpectedSize / sizeof(ulong))); input.CopyTo(dst); + + if (!littleEndian) + { + dst.Reverse(); + } } public uint256(ulong b) From a177a4170f634430ab77313a7ce9977c434fc226 Mon Sep 17 00:00:00 2001 From: dangershony Date: Sat, 11 Apr 2020 01:46:43 +0100 Subject: [PATCH 4/6] Store proven header in a new field --- .../BlockStoreBehavior.cs | 10 ++++----- .../ProvenHeadersBlockStoreBehavior.cs | 6 ++--- .../ProvenHeadersBlockStoreSignaled.cs | 4 ++-- .../ProvenHeadersConsensusManagerBehavior.cs | 4 +++- .../ProvenHeaderCoinstakeRule.cs | 6 ++--- .../ProvenHeaderRules/ProvenHeaderRuleBase.cs | 17 ++++---------- .../Behaviors/PoABlockStoreBehavior.cs | 6 ++--- src/Blockcore/Base/ChainRepository.cs | 17 ++++++-------- .../Primitives/ChainedHeaderBlock.cs | 4 ++-- src/NBitcoin/ChainedHeader.cs | 22 +++++++++++++++---- src/NBitcoin/MemoryHeaderStore.cs | 5 +++++ 11 files changed, 55 insertions(+), 46 deletions(-) diff --git a/src/Blockcore.Features.BlockStore/BlockStoreBehavior.cs b/src/Blockcore.Features.BlockStore/BlockStoreBehavior.cs index af848da58..9052ec190 100644 --- a/src/Blockcore.Features.BlockStore/BlockStoreBehavior.cs +++ b/src/Blockcore.Features.BlockStore/BlockStoreBehavior.cs @@ -357,7 +357,7 @@ public async Task AnnounceBlocksAsync(List blocksToAnnounce) this.logger.LogDebug("Block propagation preferences of the peer '{0}': prefer headers - {1}, prefer headers and IDs - {2}, will{3} revert to 'inv' now.", peer.RemoteSocketEndpoint, this.PreferHeaders, this.preferHeaderAndIDs, revertToInv ? "" : " NOT"); - var headers = new List(); + var headers = new List(); var inventoryBlockToSend = new List(); try @@ -373,7 +373,7 @@ public async Task AnnounceBlocksAsync(List blocksToAnnounce) // We expect peer to answer with getheaders message. if (bestSentHeader == null) { - await peer.SendMessageAsync(this.BuildHeadersAnnouncePayload(new[] { blocksToAnnounce.Last().Header })).ConfigureAwait(false); + await peer.SendMessageAsync(this.BuildHeadersAnnouncePayload(new[] { blocksToAnnounce.Last() })).ConfigureAwait(false); this.logger.LogTrace("(-)[SENT_SINGLE_HEADER]"); return; @@ -409,7 +409,7 @@ public async Task AnnounceBlocksAsync(List blocksToAnnounce) } // If we reached here then it means that we've found starting header. - headers.Add(chainedHeader.Header); + headers.Add(chainedHeader); } } @@ -480,9 +480,9 @@ public async Task AnnounceBlocksAsync(List blocksToAnnounce) /// /// The instance to announce to the peer. /// - protected virtual Payload BuildHeadersAnnouncePayload(IEnumerable headers) + protected virtual Payload BuildHeadersAnnouncePayload(IEnumerable headers) { - return new HeadersPayload(headers); + return new HeadersPayload(headers.Select(b => b.Header)); } public override object Clone() diff --git a/src/Blockcore.Features.BlockStore/ProvenHeadersBlockStoreBehavior.cs b/src/Blockcore.Features.BlockStore/ProvenHeadersBlockStoreBehavior.cs index ce57a8b01..a64a405cf 100644 --- a/src/Blockcore.Features.BlockStore/ProvenHeadersBlockStoreBehavior.cs +++ b/src/Blockcore.Features.BlockStore/ProvenHeadersBlockStoreBehavior.cs @@ -25,13 +25,13 @@ public ProvenHeadersBlockStoreBehavior(Network network, ChainIndexer chainIndexe /// /// The instance to announce to the peer, or if the peers requires it. - protected override Payload BuildHeadersAnnouncePayload(IEnumerable headers) + protected override Payload BuildHeadersAnnouncePayload(IEnumerable headers) { // Sanity check. That should never happen. - if (!headers.All(x => x is ProvenBlockHeader)) + if (!headers.All(x => x.ProvenBlockHeader != null)) throw new BlockStoreException("UnexpectedError: BlockHeader is expected to be a ProvenBlockHeader"); - var provenHeadersPayload = new ProvenHeadersPayload(headers.Cast().ToArray()); + var provenHeadersPayload = new ProvenHeadersPayload(headers.Select(s => s.ProvenBlockHeader).ToArray()); return provenHeadersPayload; } diff --git a/src/Blockcore.Features.BlockStore/ProvenHeadersBlockStoreSignaled.cs b/src/Blockcore.Features.BlockStore/ProvenHeadersBlockStoreSignaled.cs index 6710bcd22..6be8cdff0 100644 --- a/src/Blockcore.Features.BlockStore/ProvenHeadersBlockStoreSignaled.cs +++ b/src/Blockcore.Features.BlockStore/ProvenHeadersBlockStoreSignaled.cs @@ -44,13 +44,13 @@ protected override void AddBlockToQueue(ChainedHeaderBlock blockPair, bool isIBD { int blockHeight = blockPair.ChainedHeader.Height; - if (blockPair.ChainedHeader.Header is ProvenBlockHeader phHeader) + if (blockPair.ChainedHeader.ProvenBlockHeader != null) { this.logger.LogDebug("Current header is already a Proven Header."); // Add to the store, to be sure we actually store it anyway. // It's ProvenBlockHeaderStore responsibility to prevent us to store it twice. - this.provenBlockHeaderStore.AddToPendingBatch(phHeader, new HashHeightPair(phHeader.GetHash(), blockHeight)); + this.provenBlockHeaderStore.AddToPendingBatch(blockPair.ChainedHeader.ProvenBlockHeader, new HashHeightPair(blockPair.ChainedHeader.HashBlock, blockHeight)); } else { diff --git a/src/Blockcore.Features.Consensus/Behaviors/ProvenHeadersConsensusManagerBehavior.cs b/src/Blockcore.Features.Consensus/Behaviors/ProvenHeadersConsensusManagerBehavior.cs index ae5fa0dec..ed67f4f14 100644 --- a/src/Blockcore.Features.Consensus/Behaviors/ProvenHeadersConsensusManagerBehavior.cs +++ b/src/Blockcore.Features.Consensus/Behaviors/ProvenHeadersConsensusManagerBehavior.cs @@ -167,7 +167,9 @@ protected override Payload ConstructHeadersPayload(GetHeadersPayload getHeadersP for (int heightIndex = header.Height; heightIndex > fork.Height; heightIndex--) { - if (!(header.Header is ProvenBlockHeader provenBlockHeader)) + ProvenBlockHeader provenBlockHeader = null; + + if (header.ProvenBlockHeader == null) { provenBlockHeader = this.provenBlockHeaderStore.GetAsync(header.Height).GetAwaiter().GetResult(); diff --git a/src/Blockcore.Features.Consensus/Rules/ProvenHeaderRules/ProvenHeaderCoinstakeRule.cs b/src/Blockcore.Features.Consensus/Rules/ProvenHeaderRules/ProvenHeaderCoinstakeRule.cs index 3cad73607..afeee029a 100644 --- a/src/Blockcore.Features.Consensus/Rules/ProvenHeaderRules/ProvenHeaderCoinstakeRule.cs +++ b/src/Blockcore.Features.Consensus/Rules/ProvenHeaderRules/ProvenHeaderCoinstakeRule.cs @@ -121,7 +121,7 @@ private UnspentOutput GetAndValidatePreviousUtxo(ProvenBlockHeader header, PosRu UnspentOutput prevUtxo = null; - FetchCoinsResponse coins = this.PosParent.UtxoSet.FetchCoins(new[] {txIn.PrevOut}); + FetchCoinsResponse coins = this.PosParent.UtxoSet.FetchCoins(new[] { txIn.PrevOut }); prevUtxo = coins.UnspentOutputs[txIn.PrevOut]; if (prevUtxo?.Coins == null) { @@ -245,7 +245,7 @@ private uint256 GetPreviousStakeModifier(ChainedHeader chainedHeader) return this.LastCheckpoint.StakeModifierV2; } - var previousProvenHeader = chainedHeader.Previous.Header as ProvenBlockHeader; + var previousProvenHeader = chainedHeader.Previous.ProvenBlockHeader; if (previousProvenHeader != null) { if (previousProvenHeader.StakeModifierV2 == null) @@ -390,4 +390,4 @@ private OutPoint GetPreviousOut(ProvenBlockHeader header) return prevOut; } } -} +} \ No newline at end of file diff --git a/src/Blockcore.Features.Consensus/Rules/ProvenHeaderRules/ProvenHeaderRuleBase.cs b/src/Blockcore.Features.Consensus/Rules/ProvenHeaderRules/ProvenHeaderRuleBase.cs index 7deb17e13..8a34d88c5 100644 --- a/src/Blockcore.Features.Consensus/Rules/ProvenHeaderRules/ProvenHeaderRuleBase.cs +++ b/src/Blockcore.Features.Consensus/Rules/ProvenHeaderRules/ProvenHeaderRuleBase.cs @@ -16,6 +16,7 @@ public abstract class ProvenHeaderRuleBase : HeaderValidationConsensusRule { /// Allow access to the POS parent. protected PosConsensusRuleEngine PosParent; + protected int LastCheckpointHeight; protected CheckpointInfo LastCheckpoint; @@ -43,16 +44,6 @@ public bool IsProvenHeaderActivated(int height) return height > this.LastCheckpointHeight; } - /// - /// Determines whether header is a proven header. - /// - /// The block header. - /// true if header is a . - public bool IsProvenHeader(BlockHeader header) - { - return header is ProvenBlockHeader; - } - /// public override void Run(RuleContext context) { @@ -72,7 +63,7 @@ public override void Run(RuleContext context) return; } - if (!this.IsProvenHeader(chainedHeader.Header)) + if (chainedHeader.ProvenBlockHeader == null) { // We skip validation if the header is a regular header // This is to allow white-listed peers to sync using regular headers. @@ -80,7 +71,7 @@ public override void Run(RuleContext context) return; } - this.ProcessRule((PosRuleContext)context, chainedHeader, (ProvenBlockHeader)chainedHeader.Header); + this.ProcessRule((PosRuleContext)context, chainedHeader, chainedHeader.ProvenBlockHeader); } /// @@ -91,4 +82,4 @@ public override void Run(RuleContext context) /// The Proven Header to be validated. protected abstract void ProcessRule(PosRuleContext context, ChainedHeader chainedHeader, ProvenBlockHeader header); } -} +} \ No newline at end of file diff --git a/src/Blockcore.Features.PoA/Behaviors/PoABlockStoreBehavior.cs b/src/Blockcore.Features.PoA/Behaviors/PoABlockStoreBehavior.cs index 066784d0c..5bfb7245e 100644 --- a/src/Blockcore.Features.PoA/Behaviors/PoABlockStoreBehavior.cs +++ b/src/Blockcore.Features.PoA/Behaviors/PoABlockStoreBehavior.cs @@ -19,9 +19,9 @@ public PoABlockStoreBehavior(ChainIndexer chainIndexer, IChainState chainState, } /// - protected override Payload BuildHeadersAnnouncePayload(IEnumerable headers) + protected override Payload BuildHeadersAnnouncePayload(IEnumerable headers) { - var poaHeaders = headers.Cast().ToList(); + var poaHeaders = headers.Select(s => s.Header).Cast().ToList(); return new PoAHeadersPayload(poaHeaders); } @@ -37,4 +37,4 @@ public override object Clone() return res; } } -} +} \ No newline at end of file diff --git a/src/Blockcore/Base/ChainRepository.cs b/src/Blockcore/Base/ChainRepository.cs index ab1163526..cbbe59a21 100644 --- a/src/Blockcore/Base/ChainRepository.cs +++ b/src/Blockcore/Base/ChainRepository.cs @@ -65,7 +65,10 @@ public Task LoadAsync(ChainedHeader genesisHeader) Row firstRow = transaction.Select("Chain", 0); if (!firstRow.Exists) + { + genesisHeader.SetBlockHeaderStore(this.blockHeaderStore); return genesisHeader; + } BlockHeader nextHeader = this.dBreezeSerializer.Deserialize(firstRow.Value); Guard.Assert(nextHeader.GetHash() == genesisHeader.HashBlock); // can't swap networks @@ -85,7 +88,10 @@ public Task LoadAsync(ChainedHeader genesisHeader) tip = new ChainedHeader(nextHeader, nextHeader.GetHash(), tip); if (tip == null) + { + genesisHeader.SetBlockHeaderStore(this.blockHeaderStore); tip = genesisHeader; + } this.locator = tip.GetLocator(); return tip; @@ -122,16 +128,7 @@ public Task SaveAsync(ChainIndexer chainIndexer) BlockHeader header = block.Header; if (header is ProvenBlockHeader) { - // copy the header parameters, untill we dont make PH a normal header we store it in its own repo. - BlockHeader newHeader = chainIndexer.Network.Consensus.ConsensusFactory.CreateBlockHeader(); - newHeader.Bits = header.Bits; - newHeader.Time = header.Time; - newHeader.Nonce = header.Nonce; - newHeader.Version = header.Version; - newHeader.HashMerkleRoot = header.HashMerkleRoot; - newHeader.HashPrevBlock = header.HashPrevBlock; - - header = newHeader; + throw new Exception("Header shouldn to be ProvenBlockHeader"); } transaction.Insert("Chain", block.Height, this.dBreezeSerializer.Serialize(header)); diff --git a/src/Blockcore/Primitives/ChainedHeaderBlock.cs b/src/Blockcore/Primitives/ChainedHeaderBlock.cs index 583081c27..5ef44d6f7 100644 --- a/src/Blockcore/Primitives/ChainedHeaderBlock.cs +++ b/src/Blockcore/Primitives/ChainedHeaderBlock.cs @@ -38,11 +38,11 @@ public override string ToString() /// /// The new header to set. /// Use this method very carefully because it could cause race conditions if used at the wrong moment. - public void SetHeader(BlockHeader newHeader) + public void SetHeader(ProvenBlockHeader newHeader) { Guard.NotNull(newHeader, nameof(newHeader)); this.ChainedHeader.SetHeader(newHeader); } } -} +} \ No newline at end of file diff --git a/src/NBitcoin/ChainedHeader.cs b/src/NBitcoin/ChainedHeader.cs index 0c461c348..dee8f3971 100644 --- a/src/NBitcoin/ChainedHeader.cs +++ b/src/NBitcoin/ChainedHeader.cs @@ -85,6 +85,15 @@ public BlockHeader Header } } + /// + /// Represents a proof os stake network proven block header. + /// + /// + /// This is used only on POS networks, an should be short lived, + /// after consensus the header should be set to null and loaded from proven header store. + /// + public ProvenBlockHeader ProvenBlockHeader { get; set; } + /// Integer representation of the . /// The chain work field is represented as a byte array to reduce the memory foot print of a BigInteger private byte[] chainWork; @@ -178,6 +187,9 @@ public ChainedHeader(BlockHeader header, uint256 headerHash, ChainedHeader previ } this.CalculateChainWork(); + + if (header is ProvenBlockHeader) + this.ProvenBlockHeader = (ProvenBlockHeader)header; } public ChainedHeader(BlockHeader header, uint256 headerHash, int height) : this(header, headerHash) @@ -199,6 +211,9 @@ public ChainedHeader(BlockHeader header, uint256 headerHash, int height) : this( } this.CalculateChainWork(); + + if (header is ProvenBlockHeader) + this.ProvenBlockHeader = (ProvenBlockHeader)header; } /// @@ -313,7 +328,7 @@ public IEnumerable EnumerateToGenesis() /// public override string ToString() { - return this.Height + "-" + this.HashBlock + "-" + this.BlockValidationState + (this.Header is ProvenBlockHeader ? " - PH" : string.Empty); + return this.Height + "-" + this.HashBlock + "-" + this.BlockValidationState + (this.ProvenBlockHeader != null ? " - PH" : string.Empty); } /// @@ -697,10 +712,9 @@ private int InvertLowestOne(int n) /// /// The new header to set. /// Use this method very carefully because it could cause race conditions if used at the wrong moment. - public void SetHeader(BlockHeader newHeader) + public void SetHeader(ProvenBlockHeader newHeader) { - //this.Header = newHeader; - this.HeaderStore.StoreHeader(newHeader); + this.ProvenBlockHeader = newHeader; } } } \ No newline at end of file diff --git a/src/NBitcoin/MemoryHeaderStore.cs b/src/NBitcoin/MemoryHeaderStore.cs index 42c98b2d2..1e3fb7b00 100644 --- a/src/NBitcoin/MemoryHeaderStore.cs +++ b/src/NBitcoin/MemoryHeaderStore.cs @@ -34,6 +34,11 @@ public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) public bool StoreHeader(BlockHeader blockHeader) { + if (blockHeader is ProvenBlockHeader) + { + throw new Exception("Header can not be of type 'ProvenBlockHeader'"); + } + return this.headers.TryAdd(blockHeader.GetHash(), blockHeader); } } From aa6bba2a98cf59c167237d99a8d045b322d7155a Mon Sep 17 00:00:00 2001 From: dangershony Date: Sat, 11 Apr 2020 22:07:05 +0100 Subject: [PATCH 5/6] Fixing tests --- .../InitialBlockDownloadTest.cs | 6 +++--- .../Rules/CommonRules/PowCoinViewRuleTests.cs | 2 +- .../StakeValidatorTests.cs | 8 ++++---- .../ProvenHeadersConsensusManagerBehavior.cs | 4 ++-- .../Controllers/MiningControllerTest.cs | 12 ++++++++---- src/Blockcore.Features.PoA.Tests/PoATestsBase.cs | 3 ++- .../SlotsManagerTests.cs | 8 +++++--- src/Blockcore/Base/ChainRepository.cs | 11 ++++++++++- src/NBitcoin/ChainedHeader.cs | 13 ++++--------- src/NBitcoin/MemoryHeaderStore.cs | 5 ----- 10 files changed, 39 insertions(+), 33 deletions(-) diff --git a/src/Blockcore.Features.Consensus.Tests/InitialBlockDownloadTest.cs b/src/Blockcore.Features.Consensus.Tests/InitialBlockDownloadTest.cs index fe3d6befe..1619025e1 100644 --- a/src/Blockcore.Features.Consensus.Tests/InitialBlockDownloadTest.cs +++ b/src/Blockcore.Features.Consensus.Tests/InitialBlockDownloadTest.cs @@ -34,7 +34,7 @@ public InitialBlockDownloadTest() public void InIBDIfBehindCheckpoint() { BlockHeader blockHeader = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); - this.chainState.ConsensusTip = new ChainedHeader(blockHeader, uint256.Zero, 1000); + this.chainState.ConsensusTip = new ChainedHeader(blockHeader, blockHeader.GetHash(), 1000); var blockDownloadState = new InitialBlockDownloadState(this.chainState, this.network, this.consensusSettings, this.checkpoints, this.loggerFactory.Object, DateTimeProvider.Default); Assert.True(blockDownloadState.IsInitialBlockDownload()); } @@ -43,7 +43,7 @@ public void InIBDIfBehindCheckpoint() public void InIBDIfChainWorkIsLessThanMinimum() { BlockHeader blockHeader = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); - this.chainState.ConsensusTip = new ChainedHeader(blockHeader, uint256.Zero, this.checkpoints.GetLastCheckpointHeight() + 1); + this.chainState.ConsensusTip = new ChainedHeader(blockHeader, blockHeader.GetHash(), this.checkpoints.GetLastCheckpointHeight() + 1); var blockDownloadState = new InitialBlockDownloadState(this.chainState, this.network, this.consensusSettings, this.checkpoints, this.loggerFactory.Object, DateTimeProvider.Default); Assert.True(blockDownloadState.IsInitialBlockDownload()); } @@ -59,7 +59,7 @@ public void InIBDIfTipIsOlderThanMaxAge() // Block has a time sufficiently in the past that it can't be the tip. blockHeader.Time = ((uint) DateTimeOffset.Now.ToUnixTimeSeconds()) - (uint) this.network.MaxTipAge - 1; - this.chainState.ConsensusTip = new ChainedHeader(blockHeader, uint256.Zero, this.checkpoints.GetLastCheckpointHeight() + 1); + this.chainState.ConsensusTip = new ChainedHeader(blockHeader, blockHeader.GetHash(), this.checkpoints.GetLastCheckpointHeight() + 1); var blockDownloadState = new InitialBlockDownloadState(this.chainState, this.network, this.consensusSettings, this.checkpoints, this.loggerFactory.Object, DateTimeProvider.Default); Assert.True(blockDownloadState.IsInitialBlockDownload()); } diff --git a/src/Blockcore.Features.Consensus.Tests/Rules/CommonRules/PowCoinViewRuleTests.cs b/src/Blockcore.Features.Consensus.Tests/Rules/CommonRules/PowCoinViewRuleTests.cs index 0914d6b8e..2cb26fe66 100644 --- a/src/Blockcore.Features.Consensus.Tests/Rules/CommonRules/PowCoinViewRuleTests.cs +++ b/src/Blockcore.Features.Consensus.Tests/Rules/CommonRules/PowCoinViewRuleTests.cs @@ -79,7 +79,7 @@ private void AndARuleContext() this.ruleContext = new PowRuleContext { }; this.ruleContext.ValidationContext = new ValidationContext(); BlockHeader blockHeader = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); - this.ruleContext.ValidationContext.ChainedHeaderToValidate = new ChainedHeader(blockHeader, new uint256("bcd7d5de8d3bcc7b15e7c8e5fe77c0227cdfa6c682ca13dcf4910616f10fdd06"), HeightOfBlockchain); + this.ruleContext.ValidationContext.ChainedHeaderToValidate = new ChainedHeader(blockHeader, blockHeader.GetHash(), HeightOfBlockchain); Block block = this.network.CreateBlock(); block.Transactions = new List(); diff --git a/src/Blockcore.Features.Consensus.Tests/StakeValidatorTests.cs b/src/Blockcore.Features.Consensus.Tests/StakeValidatorTests.cs index 7993acc5c..c467a61b3 100644 --- a/src/Blockcore.Features.Consensus.Tests/StakeValidatorTests.cs +++ b/src/Blockcore.Features.Consensus.Tests/StakeValidatorTests.cs @@ -1196,7 +1196,7 @@ public void GetTargetDepthRequired_Testnet_HeightBelowMinConfirmationHeight_Uses var height = PosConsensusOptions.CoinstakeMinConfirmationActivationHeightTestnet - 2; BlockHeader blockHeader = this.Network.Consensus.ConsensusFactory.CreateBlockHeader(); - ChainedHeader header = new ChainedHeader(blockHeader, uint256.One, height); + ChainedHeader header = new ChainedHeader(blockHeader, blockHeader.GetHash(), height); var depth = this.stakeValidator.GetTargetDepthRequired(header); @@ -1211,7 +1211,7 @@ public void GetTargetDepthRequired_Testnet_HeightAtMinConfirmationHeight_UsesCha var height = PosConsensusOptions.CoinstakeMinConfirmationActivationHeightTestnet - 1; BlockHeader blockHeader = this.Network.Consensus.ConsensusFactory.CreateBlockHeader(); - ChainedHeader header = new ChainedHeader(blockHeader, uint256.One, height); + ChainedHeader header = new ChainedHeader(blockHeader, blockHeader.GetHash(), height); var depth = this.stakeValidator.GetTargetDepthRequired(header); @@ -1226,7 +1226,7 @@ public void GetTargetDepthRequired_Mainnet_HeightBelowMinConfirmationHeight_Uses var height = PosConsensusOptions.CoinstakeMinConfirmationActivationHeightMainnet - 2; BlockHeader blockHeader = this.Network.Consensus.ConsensusFactory.CreateBlockHeader(); - ChainedHeader header = new ChainedHeader(blockHeader, uint256.One, height); + ChainedHeader header = new ChainedHeader(blockHeader, blockHeader.GetHash(), height); var depth = this.stakeValidator.GetTargetDepthRequired(header); @@ -1241,7 +1241,7 @@ public void GetTargetDepthRequired_Mainnet_HeightAtMinConfirmationHeight_UsesCha var height = PosConsensusOptions.CoinstakeMinConfirmationActivationHeightMainnet - 1; BlockHeader blockHeader = this.Network.Consensus.ConsensusFactory.CreateBlockHeader(); - ChainedHeader header = new ChainedHeader(blockHeader, uint256.One, height); + ChainedHeader header = new ChainedHeader(blockHeader, blockHeader.GetHash(), height); var depth = this.stakeValidator.GetTargetDepthRequired(header); diff --git a/src/Blockcore.Features.Consensus/Behaviors/ProvenHeadersConsensusManagerBehavior.cs b/src/Blockcore.Features.Consensus/Behaviors/ProvenHeadersConsensusManagerBehavior.cs index ed67f4f14..ca8b9fe11 100644 --- a/src/Blockcore.Features.Consensus/Behaviors/ProvenHeadersConsensusManagerBehavior.cs +++ b/src/Blockcore.Features.Consensus/Behaviors/ProvenHeadersConsensusManagerBehavior.cs @@ -167,9 +167,9 @@ protected override Payload ConstructHeadersPayload(GetHeadersPayload getHeadersP for (int heightIndex = header.Height; heightIndex > fork.Height; heightIndex--) { - ProvenBlockHeader provenBlockHeader = null; + ProvenBlockHeader provenBlockHeader = header.ProvenBlockHeader; - if (header.ProvenBlockHeader == null) + if (provenBlockHeader == null) { provenBlockHeader = this.provenBlockHeaderStore.GetAsync(header.Height).GetAwaiter().GetResult(); diff --git a/src/Blockcore.Features.Miner.Tests/Controllers/MiningControllerTest.cs b/src/Blockcore.Features.Miner.Tests/Controllers/MiningControllerTest.cs index 4c107e46d..0f1ae5f78 100644 --- a/src/Blockcore.Features.Miner.Tests/Controllers/MiningControllerTest.cs +++ b/src/Blockcore.Features.Miner.Tests/Controllers/MiningControllerTest.cs @@ -48,7 +48,8 @@ public MiningControllerTest() public void Generate_With_Incorrect_Block_Count_ReturnsInvalidRequest(int? blockCount) { var consensusManager = new Mock(); - consensusManager.Setup(cm => cm.Tip).Returns(new ChainedHeader(this.network.Consensus.ConsensusFactory.CreateBlockHeader(), new uint256(0), this.network.Consensus.LastPOWBlock - 1)); + var header = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); + consensusManager.Setup(cm => cm.Tip).Returns(new ChainedHeader(header, header.GetHash(), this.network.Consensus.LastPOWBlock - 1)); var controller = new MiningController(consensusManager.Object, this.fullNode.Object, this.loggerFactory, this.network, new Mock().Object, new Mock().Object); @@ -70,7 +71,8 @@ public void Generate_With_Incorrect_Block_Count_ReturnsInvalidRequest(int? block public void Generate_Blocks_When_Model_Is_Invalid_ReturnsBadRequest() { var consensusManager = new Mock(); - consensusManager.Setup(cm => cm.Tip).Returns(new ChainedHeader(this.network.Consensus.ConsensusFactory.CreateBlockHeader(), new uint256(0), this.network.Consensus.LastPOWBlock - 1)); + var header = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); + consensusManager.Setup(cm => cm.Tip).Returns(new ChainedHeader(header, header.GetHash(), this.network.Consensus.LastPOWBlock - 1)); var controller = new MiningController(consensusManager.Object, this.fullNode.Object, this.loggerFactory, this.network, new Mock().Object, new Mock().Object); controller.ModelState.AddModelError("key", "error message"); @@ -121,7 +123,8 @@ public void GenerateBlocksOn_PowNetwork_ReturnsSuccess() public void GenerateBlocksOn_PosNetwork_ConsensusTip_IsBeforeLastPowBlock_ReturnsSuccess() { var consensusManager = new Mock(); - consensusManager.Setup(cm => cm.Tip).Returns(new ChainedHeader(this.network.Consensus.ConsensusFactory.CreateBlockHeader(), new uint256(0), this.network.Consensus.LastPOWBlock - 1)); + var header = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); + consensusManager.Setup(cm => cm.Tip).Returns(new ChainedHeader(header, header.GetHash(), this.network.Consensus.LastPOWBlock - 1)); var walletManager = new Mock(); walletManager.Setup(f => f.GetWalletsNames()).Returns(new List { wallet }); @@ -150,7 +153,8 @@ public void GenerateBlocksOn_PosNetwork_ConsensusTip_IsBeforeLastPowBlock_Return public void GenerateBlocksOn_PosNetwork_ConsensusTip_IsAfterLastPowBlock_ReturnsError() { var consensusManager = new Mock(); - consensusManager.Setup(cm => cm.Tip).Returns(new ChainedHeader(this.network.Consensus.ConsensusFactory.CreateBlockHeader(), new uint256(0), this.network.Consensus.LastPOWBlock + 1)); + var header = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); + consensusManager.Setup(cm => cm.Tip).Returns(new ChainedHeader(header, header.GetHash(), this.network.Consensus.LastPOWBlock + 1)); this.fullNode.Setup(i => i.NodeService(false)).Returns(consensusManager.Object); var controller = new MiningController(consensusManager.Object, this.fullNode.Object, this.loggerFactory, this.network, new Mock().Object, new Mock().Object); diff --git a/src/Blockcore.Features.PoA.Tests/PoATestsBase.cs b/src/Blockcore.Features.PoA.Tests/PoATestsBase.cs index 636062055..5b361d7be 100644 --- a/src/Blockcore.Features.PoA.Tests/PoATestsBase.cs +++ b/src/Blockcore.Features.PoA.Tests/PoATestsBase.cs @@ -56,7 +56,8 @@ public PoATestsBase(TestPoANetwork network = null) this.federationManager = CreateFederationManager(this, this.network, this.loggerFactory, this.signals); this.chainIndexerMock = new Mock(); - this.chainIndexerMock.Setup(x => x.Tip).Returns(new ChainedHeader(new BlockHeader(), 0, 0)); + var header = new BlockHeader(); + this.chainIndexerMock.Setup(x => x.Tip).Returns(new ChainedHeader(header, header.GetHash(), 0)); this.slotsManager = new SlotsManager(this.network, this.federationManager, this.chainIndexerMock.Object, this.loggerFactory); this.poaHeaderValidator = new PoABlockHeaderValidator(this.loggerFactory); diff --git a/src/Blockcore.Features.PoA.Tests/SlotsManagerTests.cs b/src/Blockcore.Features.PoA.Tests/SlotsManagerTests.cs index 198df43ad..35c3ae25d 100644 --- a/src/Blockcore.Features.PoA.Tests/SlotsManagerTests.cs +++ b/src/Blockcore.Features.PoA.Tests/SlotsManagerTests.cs @@ -64,7 +64,8 @@ public void GetMiningTimestamp() this.network = new TestPoANetwork(new List() { tool.GeneratePrivateKey().PubKey, key.PubKey, tool.GeneratePrivateKey().PubKey }); IFederationManager fedManager = PoATestsBase.CreateFederationManager(this, this.network, new ExtendedLoggerFactory(), new Signals.Signals(new LoggerFactory(), null)); - this.chainIndexer.Setup(x => x.Tip).Returns(new ChainedHeader(new BlockHeader(), 0, 0)); + var header = new BlockHeader(); + this.chainIndexer.Setup(x => x.Tip).Returns(new ChainedHeader(header, header.GetHash(), 0)); this.slotsManager = new SlotsManager(this.network, fedManager, this.chainIndexer.Object, new LoggerFactory()); List federationMembers = fedManager.GetFederationMembers(); @@ -93,10 +94,11 @@ public void GetMiningTimestamp() Assert.Equal(thisTurnTimestamp, this.slotsManager.GetMiningTimestamp(thisTurnTimestamp + 1)); // If we are only just past our last timestamp, but we've already mined a block there, then get the NEXT turn's timestamp. - this.chainIndexer.Setup(x => x.Tip).Returns(new ChainedHeader(new BlockHeader + header = new BlockHeader { Time = thisTurnTimestamp - }, 0, 0)); + }; + this.chainIndexer.Setup(x => x.Tip).Returns(new ChainedHeader(header, header.GetHash(), 0)); this.slotsManager = new SlotsManager(this.network, fedManager, this.chainIndexer.Object, new LoggerFactory()); Assert.Equal(nextTurnTimestamp, this.slotsManager.GetMiningTimestamp(thisTurnTimestamp + 1)); diff --git a/src/Blockcore/Base/ChainRepository.cs b/src/Blockcore/Base/ChainRepository.cs index cbbe59a21..b04402360 100644 --- a/src/Blockcore/Base/ChainRepository.cs +++ b/src/Blockcore/Base/ChainRepository.cs @@ -128,7 +128,16 @@ public Task SaveAsync(ChainIndexer chainIndexer) BlockHeader header = block.Header; if (header is ProvenBlockHeader) { - throw new Exception("Header shouldn to be ProvenBlockHeader"); + // copy the header parameters, untill we dont make PH a normal header we store it in its own repo. + BlockHeader newHeader = chainIndexer.Network.Consensus.ConsensusFactory.CreateBlockHeader(); + newHeader.Bits = header.Bits; + newHeader.Time = header.Time; + newHeader.Nonce = header.Nonce; + newHeader.Version = header.Version; + newHeader.HashMerkleRoot = header.HashMerkleRoot; + newHeader.HashPrevBlock = header.HashPrevBlock; + + header = newHeader; } transaction.Insert("Chain", block.Height, this.dBreezeSerializer.Serialize(header)); diff --git a/src/NBitcoin/ChainedHeader.cs b/src/NBitcoin/ChainedHeader.cs index dee8f3971..da6c57855 100644 --- a/src/NBitcoin/ChainedHeader.cs +++ b/src/NBitcoin/ChainedHeader.cs @@ -200,20 +200,15 @@ public ChainedHeader(BlockHeader header, uint256 headerHash, int height) : this( { this.BlockDataAvailability = BlockDataAvailabilityState.BlockAvailable; this.BlockValidationState = ValidationState.FullyValidated; - - this.HeaderStore = new MemoryHeaderStore(); - this.HeaderStore.StoreHeader(header); - } - else - { - this.HeaderStore = this.Previous.HeaderStore; - this.HeaderStore.StoreHeader(header); } + this.HeaderStore = this.Previous?.HeaderStore ?? new MemoryHeaderStore(); + this.HeaderStore.StoreHeader(header); + this.CalculateChainWork(); if (header is ProvenBlockHeader) - this.ProvenBlockHeader = (ProvenBlockHeader)header; + this.ProvenBlockHeader = (ProvenBlockHeader) header; } /// diff --git a/src/NBitcoin/MemoryHeaderStore.cs b/src/NBitcoin/MemoryHeaderStore.cs index 1e3fb7b00..42c98b2d2 100644 --- a/src/NBitcoin/MemoryHeaderStore.cs +++ b/src/NBitcoin/MemoryHeaderStore.cs @@ -34,11 +34,6 @@ public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) public bool StoreHeader(BlockHeader blockHeader) { - if (blockHeader is ProvenBlockHeader) - { - throw new Exception("Header can not be of type 'ProvenBlockHeader'"); - } - return this.headers.TryAdd(blockHeader.GetHash(), blockHeader); } } From 7c6f0b7f251c51f2ed66f012bb3032cb6880629d Mon Sep 17 00:00:00 2001 From: dangershony Date: Sun, 12 Apr 2020 16:24:01 +0100 Subject: [PATCH 6/6] Better for POS increse cache count ahead --- src/Blockcore/Consensus/LeveldbHeaderStore.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Blockcore/Consensus/LeveldbHeaderStore.cs b/src/Blockcore/Consensus/LeveldbHeaderStore.cs index 1a1505805..b9f643b03 100644 --- a/src/Blockcore/Consensus/LeveldbHeaderStore.cs +++ b/src/Blockcore/Consensus/LeveldbHeaderStore.cs @@ -23,7 +23,7 @@ public LeveldbHeaderStore(Network network, DataFolder dataFolder, ChainIndexer c this.network = network; this.ChainIndexer = chainIndexer; // this.headers = new Dictionary(); - this.headers = new MemoryCountCache(501); + this.headers = new MemoryCountCache(601); this.locker = new object(); // Open a connection to a new DB and create if not found @@ -55,8 +55,8 @@ public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) blockHeader = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); blockHeader.FromBytes(bytes, this.network.Consensus.ConsensusFactory); - // If the header is 500 blocks behind tip or one block ahead cache it. - if ((chainedHeader.Height > this.ChainIndexer.Height - 500) && (chainedHeader.Height <= this.ChainIndexer.Height + 1)) + // If the header is 500 blocks behind tip or 100 blocks ahead cache it. + if ((chainedHeader.Height > this.ChainIndexer.Height - 500) && (chainedHeader.Height <= this.ChainIndexer.Height + 100)) this.headers.AddOrUpdate(hash, blockHeader); return blockHeader;