From 280097ebcce2b1621c2859244733652cbcdb4e29 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 10:42:11 +0200 Subject: [PATCH 01/42] More comments --- .../Nethermind.Core.Test/KeccakCacheTests.cs | 31 ++++ .../Nethermind.Core/Crypto/KeccakCache.cs | 164 ++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs create mode 100644 src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs new file mode 100644 index 00000000000..ba5136817f9 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using FluentAssertions; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using NUnit.Framework; + +namespace Nethermind.Core.Test +{ + [TestFixture] + public class KeccakCacheTests + { + [Test] + public void Multiple() + { + var random = new Random(13); + var bytes = new byte[31]; // misaligned length + random.NextBytes(bytes); + + ValueHash256 expected = ValueKeccak.Compute(bytes); + + for (int i = 0; i < 10; i++) + { + ValueHash256 actual = KeccakCache.Compute(bytes); + actual.Equals(expected).Should().BeTrue(); + } + } + } +} diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs new file mode 100644 index 00000000000..412d53e2b23 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Nethermind.Core.Crypto; + +public static unsafe class KeccakCache +{ + /// + /// This counts make the cache consume 8MB of continues memory + /// + private const int Count = 64 * 1024; + private const uint BucketMask = 0x0000_FFFF; + private const uint HashMask = 0xFFFF_0000; + + private static readonly Entry* Memory; + + static KeccakCache() + { + const UIntPtr size = Count * Entry.Size; + + // Aligned, so that no torn reads if fields of Entry are properly aligned. + Memory = (Entry*)NativeMemory.AlignedAlloc(size, Entry.Size); + NativeMemory.Clear(Memory, size); + } + + [SkipLocalsInit] + public static ValueHash256 Compute(ReadOnlySpan input) + { + var fast = FastHash(input); + var index = fast & BucketMask; + + Debug.Assert(index is > 0 and < Count); + + uint hashAndLength = (fast & HashMask) | (ushort)input.Length; + + ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef(Memory), index); + + // Read aligned, volatile, won't be torn, check with computed + if (Volatile.Read(ref e.HashAndLength) == hashAndLength) + { + // There's a possibility of a hit, try lock. + if (Interlocked.CompareExchange(ref e.Lock, Entry.Locked, Entry.Unlocked) == Entry.Unlocked) + { + if (e.HashAndLength != hashAndLength) + { + // The value has been changed between reading and taking a lock. + // Release the lock and compute, Use Volatile.Write to release? + Interlocked.Exchange(ref e.Lock, Entry.Unlocked); + goto Compute; + } + + // Lock taken, copy to local + Entry copy = e; + + // Release the lock, potentially Volatile.Write?? + Interlocked.Exchange(ref e.Lock, Entry.Unlocked); + + // Lengths are equal, the input length can be used without any additional operation. + if (MemoryMarshal.CreateReadOnlySpan(ref copy.Payload, input.Length).SequenceEqual(input)) + { + return copy.Value; + } + } + } + + Compute: + var hash = ValueKeccak.Compute(input); + + // Try lock and memoize + if (Interlocked.CompareExchange(ref e.Lock, Entry.Locked, Entry.Unlocked) == Entry.Unlocked) + { + e.HashAndLength = hashAndLength; + e.Value = hash; + + input.CopyTo(MemoryMarshal.CreateSpan(ref e.Payload, input.Length)); + + // Release the lock, potentially Volatile.Write?? + Interlocked.Exchange(ref e.Lock, Entry.Unlocked); + } + + return hash; + } + + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint FastHash(ReadOnlySpan input) + { + uint hash = 13; + var length = input.Length; + + ref var b = ref MemoryMarshal.GetReference(input); + if ((length & 1) == 1) + { + hash = b; + b = ref Unsafe.Add(ref b, 1); + length -= 1; + } + if ((length & 2) == 2) + { + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + b = ref Unsafe.Add(ref b, 2); + length -= 2; + } + if ((length & 4) == 4) + { + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + b = ref Unsafe.Add(ref b, 4); + length -= 4; + } + + while (length > 0) + { + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + b = ref Unsafe.Add(ref b, 8); + length -= 8; + } + + return hash; + } + + /// + /// An entry to cache keccak + /// + [StructLayout(LayoutKind.Explicit, Size = Size)] + private struct Entry + { + public const int Unlocked = 0; + public const int Locked = 1; + + /// + /// Should work for both ARM and x64 and be aligned. + /// + public const int Size = 128; + + private const int PayloadStart = 8; + private const int ValueStart = Size - ValueHash256.MemorySize; + private const int PayloadLength = ValueStart - PayloadStart; + + [FieldOffset(0)] + public int Lock; + + /// + /// The mix of hash and length allows for a fast comparison. + /// + [FieldOffset(4)] + public uint HashAndLength; + + [FieldOffset(PayloadStart)] + public byte Payload; + + /// + /// The actual value + /// + [FieldOffset(ValueStart)] + public ValueHash256 Value; + } +} From 7ecd7a5b3d36b2238cc57826fc25b8810242b7cb Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 11:16:17 +0200 Subject: [PATCH 02/42] comments --- .../Nethermind.Core/Crypto/KeccakCache.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 412d53e2b23..9b143c98ea5 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -10,13 +10,20 @@ namespace Nethermind.Core.Crypto; +/// +/// This is a minimalistic one-way set associative cache for Keccak values. +/// +/// It allocates only 8MB of memory to store 64k of entries. +/// No misaligned reads, requires a single CAS to lock. +/// Also, uses copying on the stack to get the entry, have it copied and release the lock ASAP. +/// public static unsafe class KeccakCache { /// /// This counts make the cache consume 8MB of continues memory /// - private const int Count = 64 * 1024; - private const uint BucketMask = 0x0000_FFFF; + private const int Count = BucketMask + 1; + private const int BucketMask = 0x0000_FFFF; private const uint HashMask = 0xFFFF_0000; private static readonly Entry* Memory; @@ -141,7 +148,7 @@ private struct Entry private const int PayloadStart = 8; private const int ValueStart = Size - ValueHash256.MemorySize; - private const int PayloadLength = ValueStart - PayloadStart; + public const int MaxPayloadLength = ValueStart - PayloadStart; [FieldOffset(0)] public int Lock; From 8dce0c70e1b24ae6e3f4fd2b5c4741be67859914 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 11:20:53 +0200 Subject: [PATCH 03/42] updated --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 9b143c98ea5..d8852a6532f 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -40,6 +40,11 @@ static KeccakCache() [SkipLocalsInit] public static ValueHash256 Compute(ReadOnlySpan input) { + if (input.Length > Entry.MaxPayloadLength) + { + return ValueKeccak.Compute(input); + } + var fast = FastHash(input); var index = fast & BucketMask; From dec33ca8731b215c30b664610093f0530b590b9f Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 11:22:35 +0200 Subject: [PATCH 04/42] EVM uses the cached keccak --- src/Nethermind/Nethermind.Evm/VirtualMachine.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index 01e891b07f5..4e8292ccc0d 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -1132,7 +1132,8 @@ private CallResult ExecuteCode Date: Fri, 16 Aug 2024 11:32:22 +0200 Subject: [PATCH 05/42] state tree --- src/Nethermind/Nethermind.State/StateTree.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.State/StateTree.cs b/src/Nethermind/Nethermind.State/StateTree.cs index 9d32c8e28ab..677f3e54bfe 100644 --- a/src/Nethermind/Nethermind.State/StateTree.cs +++ b/src/Nethermind/Nethermind.State/StateTree.cs @@ -42,14 +42,14 @@ public StateTree(ITrieStore? store, ILogManager? logManager) [DebuggerStepThrough] public Account? Get(Address address, Hash256? rootHash = null) { - ReadOnlySpan bytes = Get(ValueKeccak.Compute(address.Bytes).BytesAsSpan, rootHash); + ReadOnlySpan bytes = Get(KeccakCache.Compute(address.Bytes).BytesAsSpan, rootHash); return bytes.IsEmpty ? null : _decoder.Decode(bytes); } [DebuggerStepThrough] public bool TryGetStruct(Address address, out AccountStruct account, Hash256? rootHash = null) { - ReadOnlySpan bytes = Get(ValueKeccak.Compute(address.Bytes).BytesAsSpan, rootHash); + ReadOnlySpan bytes = Get(KeccakCache.Compute(address.Bytes).BytesAsSpan, rootHash); Rlp.ValueDecoderContext valueDecoderContext = new Rlp.ValueDecoderContext(bytes); if (bytes.IsEmpty) { @@ -69,7 +69,7 @@ public bool TryGetStruct(Address address, out AccountStruct account, Hash256? ro public void Set(Address address, Account? account) { - ValueHash256 keccak = ValueKeccak.Compute(address.Bytes); + ValueHash256 keccak = KeccakCache.Compute(address.Bytes); Set(keccak.BytesAsSpan, account is null ? null : account.IsTotallyEmpty ? EmptyAccountRlp : Rlp.Encode(account)); } From df510fd0e97b77093856cd21ba72f51536858c08 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 12:55:29 +0200 Subject: [PATCH 06/42] more tests --- .../Nethermind.Core.Test/KeccakCacheTests.cs | 37 +++++++++++++++++++ .../Nethermind.Core/Crypto/KeccakCache.cs | 14 +++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs index ba5136817f9..6fbe8fbe984 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Linq; +using System.Threading.Tasks; using FluentAssertions; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -27,5 +29,40 @@ public void Multiple() actual.Equals(expected).Should().BeTrue(); } } + + [Test] + public void Collision() + { + var colliding = new[] + { + "f8ae910727f29363002d948385ff15bc6a9bacbef13bc2afc0aa8d02749668", + "baec2065df3da176cee21714b7bfb00d0c57f37a21daf2b2d4056f67270290", + "924cc47a10ad801c74a491b19492563d2351b285ff1679e5b5264e57b13bbb", + "4fb39f7800b3a43e4e722dc6fed03b126e0125d7ca713b0558564a29903ea9", + }; + + var collisions = colliding.Length; + var array = colliding.Select(c => Bytes.FromHexString(c)).ToArray(); + var values = array.Select(a => ValueKeccak.Compute(a)).ToArray(); + + var bucket = KeccakCache.GetBucket(array[0]); + + for (int i = 1; i < collisions; i++) + { + var input = array[i]; + bucket.Should().Be(KeccakCache.GetBucket(input)); + KeccakCache.Compute(input).Should().Be(values[i]); + } + + Parallel.ForEach(array, (a, state, index) => + { + ValueHash256 v = values[index]; + + for (int i = 0; i < 100_000; i++) + { + KeccakCache.Compute(a).Should().Be(v); + } + }); + } } } diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index d8852a6532f..e4d656744c2 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -20,11 +20,11 @@ namespace Nethermind.Core.Crypto; public static unsafe class KeccakCache { /// - /// This counts make the cache consume 8MB of continues memory + /// Count is defined as a +1 over bucket mask. In the future, just change the mask as the main parameter. /// private const int Count = BucketMask + 1; private const int BucketMask = 0x0000_FFFF; - private const uint HashMask = 0xFFFF_0000; + private const uint HashMask = unchecked((uint)~BucketMask); private static readonly Entry* Memory; @@ -48,7 +48,7 @@ public static ValueHash256 Compute(ReadOnlySpan input) var fast = FastHash(input); var index = fast & BucketMask; - Debug.Assert(index is > 0 and < Count); + Debug.Assert(index < Count); uint hashAndLength = (fast & HashMask) | (ushort)input.Length; @@ -100,6 +100,11 @@ public static ValueHash256 Compute(ReadOnlySpan input) return hash; } + /// + /// Gets the bucket for tests. + /// + public static uint GetBucket(ReadOnlySpan input) => FastHash(input) & BucketMask; + [SkipLocalsInit] [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint FastHash(ReadOnlySpan input) @@ -159,7 +164,8 @@ private struct Entry public int Lock; /// - /// The mix of hash and length allows for a fast comparison. + /// The mix of hash and length allows for a fast comparison and a single volatile read. + /// The length is encoded as the low part, while the hash as the high part of uint. /// [FieldOffset(4)] public uint HashAndLength; From 154037e32f92750ef4cccf6e08ddd7842db01f91 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 14:00:51 +0200 Subject: [PATCH 07/42] special cases --- .../Nethermind.Core.Test/KeccakCacheTests.cs | 30 ++++++++++++++++++- .../Nethermind.Core/Crypto/KeccakCache.cs | 16 ++++++---- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs index 6fbe8fbe984..ea676bce25f 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers.Binary; using System.Linq; using System.Threading.Tasks; using FluentAssertions; @@ -17,19 +18,35 @@ public class KeccakCacheTests [Test] public void Multiple() { + const int spins = 10; + var random = new Random(13); var bytes = new byte[31]; // misaligned length random.NextBytes(bytes); ValueHash256 expected = ValueKeccak.Compute(bytes); - for (int i = 0; i < 10; i++) + for (int i = 0; i < spins; i++) { ValueHash256 actual = KeccakCache.Compute(bytes); actual.Equals(expected).Should().BeTrue(); } } + [Test] + public void Empty() + { + ReadOnlySpan span = ReadOnlySpan.Empty; + KeccakCache.Compute(span).Should().Be(ValueKeccak.Compute(span)); + } + + [Test] + public void Very_long() + { + ReadOnlySpan span = new byte[192]; + KeccakCache.Compute(span).Should().Be(ValueKeccak.Compute(span)); + } + [Test] public void Collision() { @@ -64,5 +81,16 @@ public void Collision() } }); } + + [Test] + public void Spin_through_all() + { + Span span = stackalloc byte[4]; + for (int i = 0; i < KeccakCache.Count; i++) + { + BinaryPrimitives.WriteInt32LittleEndian(span, i); + KeccakCache.Compute(span).Should().Be(ValueKeccak.Compute(span)); + } + } } } diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index e4d656744c2..e2ab3d0cfd6 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -16,13 +16,15 @@ namespace Nethermind.Core.Crypto; /// It allocates only 8MB of memory to store 64k of entries. /// No misaligned reads, requires a single CAS to lock. /// Also, uses copying on the stack to get the entry, have it copied and release the lock ASAP. +/// +/// Consider using to release the locks only after ensuring that semantics is right. /// public static unsafe class KeccakCache { /// /// Count is defined as a +1 over bucket mask. In the future, just change the mask as the main parameter. /// - private const int Count = BucketMask + 1; + public const int Count = BucketMask + 1; private const int BucketMask = 0x0000_FFFF; private const uint HashMask = unchecked((uint)~BucketMask); @@ -40,10 +42,12 @@ static KeccakCache() [SkipLocalsInit] public static ValueHash256 Compute(ReadOnlySpan input) { + // Special cases first + + if (input.Length == 0) + return ValueKeccak.OfAnEmptyString; if (input.Length > Entry.MaxPayloadLength) - { return ValueKeccak.Compute(input); - } var fast = FastHash(input); var index = fast & BucketMask; @@ -63,15 +67,15 @@ public static ValueHash256 Compute(ReadOnlySpan input) if (e.HashAndLength != hashAndLength) { // The value has been changed between reading and taking a lock. - // Release the lock and compute, Use Volatile.Write to release? + // Release the lock and compute. Interlocked.Exchange(ref e.Lock, Entry.Unlocked); goto Compute; } - // Lock taken, copy to local + // Local copy of 128 bytes, to release the lock as soon as possible and make a key comparison without holding it. Entry copy = e; - // Release the lock, potentially Volatile.Write?? + // Release the lock Interlocked.Exchange(ref e.Lock, Entry.Unlocked); // Lengths are equal, the input length can be used without any additional operation. From 24a595b26d4d32da25a996636c00af42e945ab74 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 14:02:35 +0200 Subject: [PATCH 08/42] Release with Volatile.Write --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index e2ab3d0cfd6..cd54c88d0fa 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -76,7 +76,7 @@ public static ValueHash256 Compute(ReadOnlySpan input) Entry copy = e; // Release the lock - Interlocked.Exchange(ref e.Lock, Entry.Unlocked); + Volatile.Write(ref e.Lock, Entry.Unlocked); // Lengths are equal, the input length can be used without any additional operation. if (MemoryMarshal.CreateReadOnlySpan(ref copy.Payload, input.Length).SequenceEqual(input)) @@ -97,8 +97,8 @@ public static ValueHash256 Compute(ReadOnlySpan input) input.CopyTo(MemoryMarshal.CreateSpan(ref e.Payload, input.Length)); - // Release the lock, potentially Volatile.Write?? - Interlocked.Exchange(ref e.Lock, Entry.Unlocked); + // Release the lock + Volatile.Write(ref e.Lock, Entry.Unlocked); } return hash; From 790f86ded8dd5f05c351543bdabe9892444eb828 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 14:08:32 +0200 Subject: [PATCH 09/42] StorageTree added --- src/Nethermind/Nethermind.State/StorageTree.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.State/StorageTree.cs b/src/Nethermind/Nethermind.State/StorageTree.cs index 472441c9e3b..b6f9148203a 100644 --- a/src/Nethermind/Nethermind.State/StorageTree.cs +++ b/src/Nethermind/Nethermind.State/StorageTree.cs @@ -5,6 +5,8 @@ using System.Collections.Frozen; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Logging; @@ -46,12 +48,15 @@ public StorageTree(IScopedTrieStore? trieStore, Hash256 rootHash, ILogManager? l TrieType = TrieType.Storage; } - private static void ComputeKey(in UInt256 index, ref Span key) + private static void ComputeKey(in UInt256 index, in Span key) { index.ToBigEndian(key); - // in situ calculation - KeccakHash.ComputeHashBytesToSpan(key, key); + ValueHash256 keyHash = KeccakCache.Compute(key); + + // Assign to update the argument + Unsafe.As>(ref MemoryMarshal.GetReference(key)) + = Unsafe.As>(ref keyHash); } [SkipLocalsInit] @@ -63,7 +68,7 @@ public byte[] Get(in UInt256 index, Hash256? storageRoot = null) } Span key = stackalloc byte[32]; - ComputeKey(index, ref key); + ComputeKey(index, key); return GetArray(key, storageRoot); } @@ -92,7 +97,7 @@ public void Set(in UInt256 index, byte[] value) else { Span key = stackalloc byte[32]; - ComputeKey(index, ref key); + ComputeKey(index, in key); SetInternal(key, value); } } From 2593734bc1cfaf5f646236099bae6a1d8a99ad99 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 14:17:51 +0200 Subject: [PATCH 10/42] comments --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index cd54c88d0fa..aade5a43640 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -14,10 +14,10 @@ namespace Nethermind.Core.Crypto; /// This is a minimalistic one-way set associative cache for Keccak values. /// /// It allocates only 8MB of memory to store 64k of entries. -/// No misaligned reads, requires a single CAS to lock. -/// Also, uses copying on the stack to get the entry, have it copied and release the lock ASAP. -/// -/// Consider using to release the locks only after ensuring that semantics is right. +/// No misaligned reads. Everything is aligned to both cache lines as well as to boundaries so no torn reads. +/// Requires a single CAS to lock and to unlock. +/// On lock failure, it just moves on with execution. +/// Uses copying on the stack to get the entry, have it copied and release the lock ASAP. This is 128 bytes to copy that quite likely will be the hit. /// public static unsafe class KeccakCache { From 805e28c3644dcc342025b1326ae3d0f236bd748d Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 14:25:01 +0200 Subject: [PATCH 11/42] one less CAS --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index aade5a43640..0e6aa7a18f4 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -68,7 +68,7 @@ public static ValueHash256 Compute(ReadOnlySpan input) { // The value has been changed between reading and taking a lock. // Release the lock and compute. - Interlocked.Exchange(ref e.Lock, Entry.Unlocked); + Volatile.Write(ref e.Lock, Entry.Unlocked); goto Compute; } From b01d503fc8e6f891323cc9732549ca8e4fa36e98 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 15:22:47 +0200 Subject: [PATCH 12/42] more go-tos --- .../Nethermind.Core/Crypto/KeccakCache.cs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 0e6aa7a18f4..851637ea23d 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -42,12 +42,20 @@ static KeccakCache() [SkipLocalsInit] public static ValueHash256 Compute(ReadOnlySpan input) { - // Special cases first + Unsafe.SkipInit(out ValueHash256 hash); + // Special cases first if (input.Length == 0) - return ValueKeccak.OfAnEmptyString; + { + hash = ValueKeccak.OfAnEmptyString; + goto Return; + } + if (input.Length > Entry.MaxPayloadLength) - return ValueKeccak.Compute(input); + { + hash = ValueKeccak.Compute(input); + goto Return; + } var fast = FastHash(input); var index = fast & BucketMask; @@ -81,13 +89,14 @@ public static ValueHash256 Compute(ReadOnlySpan input) // Lengths are equal, the input length can be used without any additional operation. if (MemoryMarshal.CreateReadOnlySpan(ref copy.Payload, input.Length).SequenceEqual(input)) { - return copy.Value; + hash = copy.Value; + goto Return; } } } Compute: - var hash = ValueKeccak.Compute(input); + hash = ValueKeccak.Compute(input); // Try lock and memoize if (Interlocked.CompareExchange(ref e.Lock, Entry.Locked, Entry.Unlocked) == Entry.Unlocked) @@ -101,6 +110,7 @@ public static ValueHash256 Compute(ReadOnlySpan input) Volatile.Write(ref e.Lock, Entry.Unlocked); } + Return: return hash; } From 0118c1d280e90ce7dabcc83d6f9e5333f4ddf7b7 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 16 Aug 2024 17:52:25 +0200 Subject: [PATCH 13/42] one less branch in hash --- .../Nethermind.Core/Crypto/KeccakCache.cs | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 851637ea23d..adda6f83559 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -123,16 +123,27 @@ public static ValueHash256 Compute(ReadOnlySpan input) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint FastHash(ReadOnlySpan input) { - uint hash = 13; + Debug.Assert(input.Length >= 1, "Cannot hash empty"); + var length = input.Length; ref var b = ref MemoryMarshal.GetReference(input); - if ((length & 1) == 1) - { - hash = b; - b = ref Unsafe.Add(ref b, 1); - length -= 1; - } + + // Start with first + uint hash = b; + + // This is done below, without branches + // if ((length & 1) == 1) + // { + // hash = b; + // b = ref Unsafe.Add(ref b, 1); + // length -= 1; + // } + + var bit = length & 1; + b = ref Unsafe.Add(ref b, bit); + length -= bit; + if ((length & 2) == 2) { hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); From 2e74b764b5f9aefd7a4a4eaff375a9a4f53cb49e Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Fri, 16 Aug 2024 17:33:24 +0100 Subject: [PATCH 14/42] Start with per instance random --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index adda6f83559..30380226dfb 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -21,6 +21,7 @@ namespace Nethermind.Core.Crypto; /// public static unsafe class KeccakCache { + private static readonly uint s_instanceRandom = (uint)System.Security.Cryptography.RandomNumberGenerator.GetInt32(int.MinValue, int.MaxValue); /// /// Count is defined as a +1 over bucket mask. In the future, just change the mask as the main parameter. /// @@ -130,7 +131,7 @@ private static uint FastHash(ReadOnlySpan input) ref var b = ref MemoryMarshal.GetReference(input); // Start with first - uint hash = b; + uint hash = s_instanceRandom ^ b; // This is done below, without branches // if ((length & 1) == 1) From a29dcec7c411f10f6001e461fa500c6248ab2270 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Fri, 16 Aug 2024 17:40:49 +0100 Subject: [PATCH 15/42] Include length in hash seed --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 30380226dfb..a0404a1956d 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -131,7 +131,7 @@ private static uint FastHash(ReadOnlySpan input) ref var b = ref MemoryMarshal.GetReference(input); // Start with first - uint hash = s_instanceRandom ^ b; + uint hash = (s_instanceRandom + (uint)length) ^ b; // This is done below, without branches // if ((length & 1) == 1) From e654d13cf0fb6b489b39e8123fcf786841664bbc Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Fri, 16 Aug 2024 17:42:10 +0100 Subject: [PATCH 16/42] Improve comment --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index a0404a1956d..530a33a9f26 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -130,7 +130,7 @@ private static uint FastHash(ReadOnlySpan input) ref var b = ref MemoryMarshal.GetReference(input); - // Start with first + // Start with instance random, length and first as seed uint hash = (s_instanceRandom + (uint)length) ^ b; // This is done below, without branches From c88d9310b000c0b6494d56281945faca161293b3 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 17 Aug 2024 12:54:40 +0100 Subject: [PATCH 17/42] Move stackalloc out of common path and inline ComputeKey to it --- .../Nethermind.State/StorageTree.cs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.State/StorageTree.cs b/src/Nethermind/Nethermind.State/StorageTree.cs index b6f9148203a..dccc0e0cbac 100644 --- a/src/Nethermind/Nethermind.State/StorageTree.cs +++ b/src/Nethermind/Nethermind.State/StorageTree.cs @@ -48,6 +48,8 @@ public StorageTree(IScopedTrieStore? trieStore, Hash256 rootHash, ILogManager? l TrieType = TrieType.Storage; } + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ComputeKey(in UInt256 index, in Span key) { index.ToBigEndian(key); @@ -67,9 +69,15 @@ public byte[] Get(in UInt256 index, Hash256? storageRoot = null) return GetArray(Lookup[index], storageRoot); } - Span key = stackalloc byte[32]; - ComputeKey(index, key); - return GetArray(key, storageRoot); + return GetWithKeyGenerate(in index, storageRoot); + + [SkipLocalsInit] + byte[] GetWithKeyGenerate(in UInt256 index, Hash256 storageRoot) + { + Span key = stackalloc byte[32]; + ComputeKey(index, key); + return GetArray(key, storageRoot); + } } public byte[] GetArray(ReadOnlySpan rawKey, Hash256? rootHash = null) @@ -95,6 +103,12 @@ public void Set(in UInt256 index, byte[] value) SetInternal(Lookup[index], value); } else + { + SetWithKeyGenerate(in index, value); + } + + [SkipLocalsInit] + void SetWithKeyGenerate(in UInt256 index, byte[] value) { Span key = stackalloc byte[32]; ComputeKey(index, in key); From 6fcfb9a7f5c2c124daa5b5067d0e6c731f87db34 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 17 Aug 2024 16:31:43 +0100 Subject: [PATCH 18/42] Unify hashing --- src/Nethermind/Nethermind.Core/Address.cs | 14 +---- .../Nethermind.Core/Crypto/Hash256.cs | 14 +---- .../Nethermind.Core/Crypto/KeccakCache.cs | 56 ++--------------- .../Nethermind.Core/Crypto/PublicKey.cs | 19 +----- .../Extensions/SpanExtensions.cs | 63 +++++++++++++++++++ src/Nethermind/Nethermind.Core/StorageCell.cs | 11 ++-- .../Nethermind.State/PreBlockCaches.cs | 28 +-------- 7 files changed, 77 insertions(+), 128 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Address.cs b/src/Nethermind/Nethermind.Core/Address.cs index 3cdab2947f8..875dd2d17ae 100644 --- a/src/Nethermind/Nethermind.Core/Address.cs +++ b/src/Nethermind/Nethermind.Core/Address.cs @@ -20,11 +20,6 @@ namespace Nethermind.Core [TypeConverter(typeof(AddressTypeConverter))] public class Address : IEquatable
, IComparable
{ - // Ensure that hashes are different for every run of the node and every node, so if are any hash collisions on - // one node they will not be the same on another node or across a restart so hash collision cannot be used to degrade - // the performance of the network as a whole. - private static readonly uint s_instanceRandom = (uint)System.Security.Cryptography.RandomNumberGenerator.GetInt32(int.MinValue, int.MaxValue); - public const int Size = 20; private const int HexCharsCount = 2 * Size; // 5a4eab120fb44eb6684e5e32785702ff45ea344d private const int PrefixedHexCharsCount = 2 + HexCharsCount; // 0x5a4eab120fb44eb6684e5e32785702ff45ea344d @@ -192,14 +187,7 @@ public override bool Equals(object? obj) return obj.GetType() == GetType() && Equals((Address)obj); } - public override int GetHashCode() - { - uint hash = s_instanceRandom; - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(Bytes))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(ulong)))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 2))); - return (int)hash; - } + public override int GetHashCode() => new ReadOnlySpan(Bytes).FastHash(); public static bool operator ==(Address? a, Address? b) { diff --git a/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs b/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs index 0f223c05197..808a262d22b 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs @@ -66,19 +66,9 @@ public ValueHash256(ReadOnlySpan bytes) public bool Equals(Hash256? other) => _bytes.Equals(other?.ValueHash256._bytes ?? default); - public override int GetHashCode() - { - return GetChainedHashCode(s_instanceRandom); - } + public override int GetHashCode() => GetChainedHashCode(s_instanceRandom); - public int GetChainedHashCode(uint previousHash) - { - uint hash = BitOperations.Crc32C(previousHash, Unsafe.As, ulong>(ref Unsafe.AsRef(in _bytes))); - hash = BitOperations.Crc32C(hash, Unsafe.Add(ref Unsafe.As, ulong>(ref Unsafe.AsRef(in _bytes)), 1)); - hash = BitOperations.Crc32C(hash, Unsafe.Add(ref Unsafe.As, ulong>(ref Unsafe.AsRef(in _bytes)), 2)); - hash = BitOperations.Crc32C(hash, Unsafe.Add(ref Unsafe.As, ulong>(ref Unsafe.AsRef(in _bytes)), 3)); - return (int)hash; - } + public int GetChainedHashCode(uint previousHash) => Bytes.FastHash() ^ (int)previousHash; public int CompareTo(ValueHash256 other) { diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 530a33a9f26..fa6695dc578 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -7,6 +7,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; +using Nethermind.Core.Extensions; namespace Nethermind.Core.Crypto; @@ -21,7 +22,6 @@ namespace Nethermind.Core.Crypto; /// public static unsafe class KeccakCache { - private static readonly uint s_instanceRandom = (uint)System.Security.Cryptography.RandomNumberGenerator.GetInt32(int.MinValue, int.MaxValue); /// /// Count is defined as a +1 over bucket mask. In the future, just change the mask as the main parameter. /// @@ -58,8 +58,8 @@ public static ValueHash256 Compute(ReadOnlySpan input) goto Return; } - var fast = FastHash(input); - var index = fast & BucketMask; + uint fast = (uint)input.FastHash(); + uint index = fast & BucketMask; Debug.Assert(index < Count); @@ -118,55 +118,7 @@ public static ValueHash256 Compute(ReadOnlySpan input) /// /// Gets the bucket for tests. /// - public static uint GetBucket(ReadOnlySpan input) => FastHash(input) & BucketMask; - - [SkipLocalsInit] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static uint FastHash(ReadOnlySpan input) - { - Debug.Assert(input.Length >= 1, "Cannot hash empty"); - - var length = input.Length; - - ref var b = ref MemoryMarshal.GetReference(input); - - // Start with instance random, length and first as seed - uint hash = (s_instanceRandom + (uint)length) ^ b; - - // This is done below, without branches - // if ((length & 1) == 1) - // { - // hash = b; - // b = ref Unsafe.Add(ref b, 1); - // length -= 1; - // } - - var bit = length & 1; - b = ref Unsafe.Add(ref b, bit); - length -= bit; - - if ((length & 2) == 2) - { - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); - b = ref Unsafe.Add(ref b, 2); - length -= 2; - } - if ((length & 4) == 4) - { - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); - b = ref Unsafe.Add(ref b, 4); - length -= 4; - } - - while (length > 0) - { - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); - b = ref Unsafe.Add(ref b, 8); - length -= 8; - } - - return hash; - } + public static uint GetBucket(ReadOnlySpan input) => (uint)input.FastHash() & BucketMask; /// /// An entry to cache keccak diff --git a/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs b/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs index c8ab15b9fdd..19db37e7914 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs @@ -16,11 +16,6 @@ namespace Nethermind.Core.Crypto [JsonConverter(typeof(PublicKeyConverter))] public class PublicKey : IEquatable { - // Ensure that hashes are different for every run of the node and every node, so if are any hash collisions on - // one node they will not be the same on another node or across a restart so hash collision cannot be used to degrade - // the performance of the network as a whole. - private static readonly uint s_instanceRandom = (uint)System.Security.Cryptography.RandomNumberGenerator.GetInt32(int.MinValue, int.MaxValue); - public const int PrefixedLengthInBytes = 65; public const int LengthInBytes = 64; private Address? _address; @@ -112,19 +107,7 @@ public override int GetHashCode() return _hashCode; } - private static int GetHashCode(byte[] bytes) - { - uint hash = s_instanceRandom; - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(bytes))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long)))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 2))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 3))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 4))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 5))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 6))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 7))); - return (int)hash; - } + private static int GetHashCode(byte[] bytes) => new ReadOnlySpan(bytes).FastHash(); public override string ToString() { diff --git a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs index 99b4b7645fe..35e58818ad2 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs @@ -4,6 +4,7 @@ using System; using System.Buffers; using System.Diagnostics; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Nethermind.Core.Collections; @@ -13,6 +14,11 @@ namespace Nethermind.Core.Extensions { public static class SpanExtensions { + // Ensure that hashes are different for every run of the node and every node, so if are any hash collisions on + // one node they will not be the same on another node or across a restart so hash collision cannot be used to degrade + // the performance of the network as a whole. + private static readonly uint s_instanceRandom = (uint)System.Security.Cryptography.RandomNumberGenerator.GetInt32(int.MinValue, int.MaxValue); + public static string ToHexString(this in ReadOnlySpan span, bool withZeroX) { return ToHexString(span, withZeroX, false, false); @@ -166,5 +172,62 @@ public static ArrayPoolList ToPooledList(this in ReadOnlySpan span) newList.AddRange(span); return newList; } + + [SkipLocalsInit] + public static int FastHash(this ReadOnlySpan input) + { + var length = input.Length; + if (length == 0) return 0; + + ref var b = ref MemoryMarshal.GetReference(input); + + // Start with instance random, length and first as seed + uint hash = (s_instanceRandom + (uint)length) ^ b; + + // This is done below, without branches + // if ((length & 1) == 1) + // { + // hash = b; + // b = ref Unsafe.Add(ref b, 1); + // length -= 1; + // } + + uint bit = (uint)length & sizeof(byte); + b = ref Unsafe.Add(ref b, bit); + length -= (int)bit; + + if ((length & sizeof(ushort)) != 0) + { + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + b = ref Unsafe.Add(ref b, sizeof(ushort)); + length -= sizeof(ushort); + } + if ((length & sizeof(uint)) != 0) + { + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + b = ref Unsafe.Add(ref b, sizeof(uint)); + length -= sizeof(uint); + } + + while (length >= sizeof(ulong) * 3) + { + // Crc32C is 3 cycle latency, 1 cycle throughput. However is complex to fully + // break the dependency chain when most of our hashes are on <= 32 byte data. + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref b, sizeof(ulong)))); + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref b, sizeof(ulong) * 2))); + b = ref Unsafe.Add(ref b, sizeof(ulong) * 3); + length -= sizeof(ulong) * 3; + } + + while (length > 0) + { + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + b = ref Unsafe.Add(ref b, sizeof(ulong)); + length -= sizeof(ulong); + } + + return (int)hash; + } } } diff --git a/src/Nethermind/Nethermind.Core/StorageCell.cs b/src/Nethermind/Nethermind.Core/StorageCell.cs index 62e73e89156..8007a51fc80 100644 --- a/src/Nethermind/Nethermind.Core/StorageCell.cs +++ b/src/Nethermind/Nethermind.Core/StorageCell.cs @@ -3,9 +3,10 @@ using System; using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Int256; namespace Nethermind.Core @@ -57,12 +58,8 @@ public override bool Equals(object? obj) public override int GetHashCode() { - uint hash = (uint)Address.GetHashCode(); - hash = BitOperations.Crc32C(hash, _index.u0); - hash = BitOperations.Crc32C(hash, _index.u1); - hash = BitOperations.Crc32C(hash, _index.u2); - hash = BitOperations.Crc32C(hash, _index.u3); - return (int)hash; + int hash = MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in _index), 1)).FastHash(); + return hash ^ Address.GetHashCode(); } public override string ToString() diff --git a/src/Nethermind/Nethermind.State/PreBlockCaches.cs b/src/Nethermind/Nethermind.State/PreBlockCaches.cs index 39a5b39e028..607d55998ec 100644 --- a/src/Nethermind/Nethermind.State/PreBlockCaches.cs +++ b/src/Nethermind/Nethermind.State/PreBlockCaches.cs @@ -8,6 +8,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Trie; namespace Nethermind.State; @@ -38,31 +39,6 @@ public readonly struct PrecompileCacheKey(Address address, ReadOnlyMemory private ReadOnlyMemory Data { get; } = data; public bool Equals(PrecompileCacheKey other) => Address == other.Address && Data.Span.SequenceEqual(other.Data.Span); public override bool Equals(object? obj) => obj is PrecompileCacheKey other && Equals(other); - public override int GetHashCode() - { - uint crc = (uint)Address.GetHashCode(); - ReadOnlySpan span = Data.Span; - var longSize = span.Length / sizeof(ulong) * sizeof(ulong); - if (longSize > 0) - { - foreach (ulong ul in MemoryMarshal.Cast(span[..longSize])) - { - crc = BitOperations.Crc32C(crc, ul); - } - foreach (byte b in span[longSize..]) - { - crc = BitOperations.Crc32C(crc, b); - } - } - else - { - foreach (byte b in span) - { - crc = BitOperations.Crc32C(crc, b); - } - } - - return (int)crc; - } + public override int GetHashCode() => Data.Span.FastHash() ^ Address.GetHashCode(); } } From 381511ec9065deb40348c59f684ea1c580a8e0ce Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 18 Aug 2024 08:17:45 +0100 Subject: [PATCH 19/42] Use full hash --- .../Nethermind.Core/Crypto/KeccakCache.cs | 47 +++++++++---------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index fa6695dc578..157d514e59c 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -3,7 +3,6 @@ using System; using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; @@ -27,7 +26,6 @@ public static unsafe class KeccakCache /// public const int Count = BucketMask + 1; private const int BucketMask = 0x0000_FFFF; - private const uint HashMask = unchecked((uint)~BucketMask); private static readonly Entry* Memory; @@ -58,34 +56,34 @@ public static ValueHash256 Compute(ReadOnlySpan input) goto Return; } - uint fast = (uint)input.FastHash(); - uint index = fast & BucketMask; + int fast = input.FastHash(); + uint index = (uint)fast & BucketMask; Debug.Assert(index < Count); - uint hashAndLength = (fast & HashMask) | (ushort)input.Length; - ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef(Memory), index); // Read aligned, volatile, won't be torn, check with computed - if (Volatile.Read(ref e.HashAndLength) == hashAndLength) + if (Volatile.Read(ref e.Hash) == fast) { // There's a possibility of a hit, try lock. - if (Interlocked.CompareExchange(ref e.Lock, Entry.Locked, Entry.Unlocked) == Entry.Unlocked) + int lockAndLength = Volatile.Read(ref e.LockAndLength); + // Lock by negating length if the length is the same size as input. + if (lockAndLength == input.Length && Interlocked.CompareExchange(ref e.LockAndLength, -lockAndLength, lockAndLength) == lockAndLength) { - if (e.HashAndLength != hashAndLength) + if (e.Hash != fast) { // The value has been changed between reading and taking a lock. // Release the lock and compute. - Volatile.Write(ref e.Lock, Entry.Unlocked); + Volatile.Write(ref e.LockAndLength, lockAndLength); goto Compute; } // Local copy of 128 bytes, to release the lock as soon as possible and make a key comparison without holding it. Entry copy = e; - // Release the lock - Volatile.Write(ref e.Lock, Entry.Unlocked); + // Release the lock, by setting back to unnegated length. + Volatile.Write(ref e.LockAndLength, lockAndLength); // Lengths are equal, the input length can be used without any additional operation. if (MemoryMarshal.CreateReadOnlySpan(ref copy.Payload, input.Length).SequenceEqual(input)) @@ -100,15 +98,18 @@ public static ValueHash256 Compute(ReadOnlySpan input) hash = ValueKeccak.Compute(input); // Try lock and memoize - if (Interlocked.CompareExchange(ref e.Lock, Entry.Locked, Entry.Unlocked) == Entry.Unlocked) + int length = Volatile.Read(ref e.LockAndLength); + // Negative value means that the entry is locked, set to int.MinValue to avoid confusion empty entry, + // since we are overwriting it anyway e.g. -0 would not be a reliable locked state. + if (length >= 0 && Interlocked.CompareExchange(ref e.LockAndLength, int.MinValue, length) == length) { - e.HashAndLength = hashAndLength; + e.Hash = fast; e.Value = hash; input.CopyTo(MemoryMarshal.CreateSpan(ref e.Payload, input.Length)); - // Release the lock - Volatile.Write(ref e.Lock, Entry.Unlocked); + // Release the lock, input.Length is always positive so setting it is enough. + Volatile.Write(ref e.LockAndLength, input.Length); } Return: @@ -126,9 +127,6 @@ public static ValueHash256 Compute(ReadOnlySpan input) [StructLayout(LayoutKind.Explicit, Size = Size)] private struct Entry { - public const int Unlocked = 0; - public const int Locked = 1; - /// /// Should work for both ARM and x64 and be aligned. /// @@ -138,15 +136,14 @@ private struct Entry private const int ValueStart = Size - ValueHash256.MemorySize; public const int MaxPayloadLength = ValueStart - PayloadStart; - [FieldOffset(0)] - public int Lock; - /// - /// The mix of hash and length allows for a fast comparison and a single volatile read. - /// The length is encoded as the low part, while the hash as the high part of uint. + /// Length is always positive so we can use a negative length to indicate that it is locked. /// + [FieldOffset(0)] + public int LockAndLength; + [FieldOffset(4)] - public uint HashAndLength; + public int Hash; [FieldOffset(PayloadStart)] public byte Payload; From 9e2ea276c606e2f85b2f1d638aacc177330443dc Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 18 Aug 2024 08:58:20 +0100 Subject: [PATCH 20/42] Faster StorageCell equality --- src/Nethermind/Nethermind.Core/StorageCell.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/StorageCell.cs b/src/Nethermind/Nethermind.Core/StorageCell.cs index 8007a51fc80..47d6451ec8f 100644 --- a/src/Nethermind/Nethermind.Core/StorageCell.cs +++ b/src/Nethermind/Nethermind.Core/StorageCell.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Int256; @@ -18,17 +19,16 @@ namespace Nethermind.Core private readonly bool _isHash; public Address Address { get; } + public bool IsHash => _isHash; public UInt256 Index => _index; public ValueHash256 Hash => _isHash ? Unsafe.As(ref Unsafe.AsRef(in _index)) : GetHash(); - public bool IsHash => _isHash; - private ValueHash256 GetHash() { Span key = stackalloc byte[32]; Index.ToBigEndian(key); - return ValueKeccak.Compute(key); + return KeccakCache.Compute(key); } public StorageCell(Address address, in UInt256 index) @@ -44,7 +44,10 @@ public StorageCell(Address address, ValueHash256 hash) _isHash = true; } - public bool Equals(StorageCell other) => Address.Equals(other.Address) && Index.Equals(other.Index); + public bool Equals(StorageCell other) => + _isHash == other._isHash && + Unsafe.As>(ref Unsafe.AsRef(in _index)) == Unsafe.As>(ref Unsafe.AsRef(in other._index)) && + Address.Equals(other.Address) && Index.Equals(other.Index); public override bool Equals(object? obj) { From c3996a2a8bde84dbb86918ea2c3663a0a4cc9f88 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 18 Aug 2024 16:52:41 +0100 Subject: [PATCH 21/42] Less copy --- .../Nethermind.Core/Crypto/KeccakCache.cs | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 157d514e59c..d4dc388c860 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -41,37 +41,37 @@ static KeccakCache() [SkipLocalsInit] public static ValueHash256 Compute(ReadOnlySpan input) { - Unsafe.SkipInit(out ValueHash256 hash); + Unsafe.SkipInit(out ValueHash256 keccak256); // Special cases first if (input.Length == 0) { - hash = ValueKeccak.OfAnEmptyString; + keccak256 = ValueKeccak.OfAnEmptyString; goto Return; } if (input.Length > Entry.MaxPayloadLength) { - hash = ValueKeccak.Compute(input); + keccak256 = ValueKeccak.Compute(input); goto Return; } - int fast = input.FastHash(); - uint index = (uint)fast & BucketMask; + int hashCode = input.FastHash(); + uint index = (uint)hashCode & BucketMask; Debug.Assert(index < Count); ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef(Memory), index); // Read aligned, volatile, won't be torn, check with computed - if (Volatile.Read(ref e.Hash) == fast) + if (Volatile.Read(ref e.HashCode) == hashCode) { // There's a possibility of a hit, try lock. int lockAndLength = Volatile.Read(ref e.LockAndLength); // Lock by negating length if the length is the same size as input. if (lockAndLength == input.Length && Interlocked.CompareExchange(ref e.LockAndLength, -lockAndLength, lockAndLength) == lockAndLength) { - if (e.Hash != fast) + if (e.HashCode != hashCode) { // The value has been changed between reading and taking a lock. // Release the lock and compute. @@ -79,23 +79,25 @@ public static ValueHash256 Compute(ReadOnlySpan input) goto Compute; } - // Local copy of 128 bytes, to release the lock as soon as possible and make a key comparison without holding it. - Entry copy = e; + // Take local copy of the payload and hash, to release the lock as soon as possible and make a key comparison without holding it. + // Local copy of 64+32 payload bytes. + Payload copy = e.Value; + // Copy Keccak256 directly to the local return variable, since we will overwrite if no match anyway. + keccak256 = e.Keccak256; // Release the lock, by setting back to unnegated length. Volatile.Write(ref e.LockAndLength, lockAndLength); // Lengths are equal, the input length can be used without any additional operation. - if (MemoryMarshal.CreateReadOnlySpan(ref copy.Payload, input.Length).SequenceEqual(input)) + if (MemoryMarshal.CreateReadOnlySpan(ref copy.Start, input.Length).SequenceEqual(input)) { - hash = copy.Value; goto Return; } } } Compute: - hash = ValueKeccak.Compute(input); + keccak256 = ValueKeccak.Compute(input); // Try lock and memoize int length = Volatile.Read(ref e.LockAndLength); @@ -103,17 +105,17 @@ public static ValueHash256 Compute(ReadOnlySpan input) // since we are overwriting it anyway e.g. -0 would not be a reliable locked state. if (length >= 0 && Interlocked.CompareExchange(ref e.LockAndLength, int.MinValue, length) == length) { - e.Hash = fast; - e.Value = hash; + e.HashCode = hashCode; + e.Keccak256 = keccak256; - input.CopyTo(MemoryMarshal.CreateSpan(ref e.Payload, input.Length)); + input.CopyTo(MemoryMarshal.CreateSpan(ref e.Value.Start, input.Length)); // Release the lock, input.Length is always positive so setting it is enough. Volatile.Write(ref e.LockAndLength, input.Length); } Return: - return hash; + return keccak256; } /// @@ -142,16 +144,28 @@ private struct Entry [FieldOffset(0)] public int LockAndLength; + /// + /// The fast Crc32c of the Value + /// [FieldOffset(4)] - public int Hash; + public int HashCode; + /// + /// The actual value + /// [FieldOffset(PayloadStart)] - public byte Payload; + public Payload Value; /// - /// The actual value + /// The Keccak of the Value /// [FieldOffset(ValueStart)] - public ValueHash256 Value; + public ValueHash256 Keccak256; + } + + [InlineArray(Entry.MaxPayloadLength)] + private struct Payload + { + public byte Start; } } From d1483ce78512938aa3d09ec4c47f04255e986cc4 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 18 Aug 2024 17:44:10 +0100 Subject: [PATCH 22/42] Even less copy --- .../Nethermind.Core/Crypto/KeccakCache.cs | 7 ++++++- src/Nethermind/Nethermind.Evm/VirtualMachine.cs | 3 ++- src/Nethermind/Nethermind.State/StateTree.cs | 2 +- src/Nethermind/Nethermind.State/StorageTree.cs | 13 ++++++------- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index d4dc388c860..afe0b8808e7 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -41,8 +41,13 @@ static KeccakCache() [SkipLocalsInit] public static ValueHash256 Compute(ReadOnlySpan input) { - Unsafe.SkipInit(out ValueHash256 keccak256); + ComputeTo(input, out ValueHash256 keccak256); + return keccak256; + } + [SkipLocalsInit] + public static ValueHash256 ComputeTo(ReadOnlySpan input, out ValueHash256 keccak256) + { // Special cases first if (input.Length == 0) { diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index 4e8292ccc0d..6761b10bd21 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -1133,7 +1133,8 @@ private CallResult ExecuteCode(ref stack.PushBytesRef())); break; } case Instruction.ADDRESS: diff --git a/src/Nethermind/Nethermind.State/StateTree.cs b/src/Nethermind/Nethermind.State/StateTree.cs index 677f3e54bfe..19cc6ca2a96 100644 --- a/src/Nethermind/Nethermind.State/StateTree.cs +++ b/src/Nethermind/Nethermind.State/StateTree.cs @@ -69,7 +69,7 @@ public bool TryGetStruct(Address address, out AccountStruct account, Hash256? ro public void Set(Address address, Account? account) { - ValueHash256 keccak = KeccakCache.Compute(address.Bytes); + KeccakCache.ComputeTo(address.Bytes, out ValueHash256 keccak); Set(keccak.BytesAsSpan, account is null ? null : account.IsTotallyEmpty ? EmptyAccountRlp : Rlp.Encode(account)); } diff --git a/src/Nethermind/Nethermind.State/StorageTree.cs b/src/Nethermind/Nethermind.State/StorageTree.cs index dccc0e0cbac..748ed3aa159 100644 --- a/src/Nethermind/Nethermind.State/StorageTree.cs +++ b/src/Nethermind/Nethermind.State/StorageTree.cs @@ -50,15 +50,14 @@ public StorageTree(IScopedTrieStore? trieStore, Hash256 rootHash, ILogManager? l [SkipLocalsInit] [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ComputeKey(in UInt256 index, in Span key) + private static void ComputeKey(in UInt256 index, Span key) { index.ToBigEndian(key); - ValueHash256 keyHash = KeccakCache.Compute(key); - - // Assign to update the argument - Unsafe.As>(ref MemoryMarshal.GetReference(key)) - = Unsafe.As>(ref keyHash); + // We can't direct ComputeTo the key as its also the input, so need a separate variable + KeccakCache.ComputeTo(key, out ValueHash256 keyHash); + // Which we can then directly assign to fast update the key + Unsafe.As(ref MemoryMarshal.GetReference(key)) = keyHash; } [SkipLocalsInit] @@ -111,7 +110,7 @@ public void Set(in UInt256 index, byte[] value) void SetWithKeyGenerate(in UInt256 index, byte[] value) { Span key = stackalloc byte[32]; - ComputeKey(index, in key); + ComputeKey(index, key); SetInternal(key, value); } } From 2070cc243219c1d12e2ab8bd92d58bbda17c705e Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 01:43:09 +0100 Subject: [PATCH 23/42] Doesn't need to return --- .../Nethermind.Core/Crypto/KeccakCache.cs | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index afe0b8808e7..9b78784a81d 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -46,19 +46,12 @@ public static ValueHash256 Compute(ReadOnlySpan input) } [SkipLocalsInit] - public static ValueHash256 ComputeTo(ReadOnlySpan input, out ValueHash256 keccak256) + public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak256) { - // Special cases first - if (input.Length == 0) + // Special cases jump forward as unpredicted + if (input.Length == 0 || input.Length > Entry.MaxPayloadLength) { - keccak256 = ValueKeccak.OfAnEmptyString; - goto Return; - } - - if (input.Length > Entry.MaxPayloadLength) - { - keccak256 = ValueKeccak.Compute(input); - goto Return; + goto Uncommon; } int hashCode = input.FastHash(); @@ -96,7 +89,8 @@ public static ValueHash256 ComputeTo(ReadOnlySpan input, out ValueHash256 // Lengths are equal, the input length can be used without any additional operation. if (MemoryMarshal.CreateReadOnlySpan(ref copy.Start, input.Length).SequenceEqual(input)) { - goto Return; + // Current keccak256 is correct hash. + return; } } } @@ -119,8 +113,15 @@ public static ValueHash256 ComputeTo(ReadOnlySpan input, out ValueHash256 Volatile.Write(ref e.LockAndLength, input.Length); } - Return: - return keccak256; + Uncommon: + if (input.Length == 0) + { + keccak256 = ValueKeccak.OfAnEmptyString; + } + else + { + keccak256 = ValueKeccak.Compute(input); + } } /// From 28e0fa456e21a80ff77f6fa04be47c7c97fee0e1 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 02:39:40 +0100 Subject: [PATCH 24/42] Update alignment comments --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 9b78784a81d..003d4fda942 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -78,7 +78,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 } // Take local copy of the payload and hash, to release the lock as soon as possible and make a key comparison without holding it. - // Local copy of 64+32 payload bytes. + // Local copy of 8+16+64 payload bytes. Payload copy = e.Value; // Copy Keccak256 directly to the local return variable, since we will overwrite if no match anyway. keccak256 = e.Keccak256; @@ -159,6 +159,7 @@ private struct Entry /// /// The actual value /// + /// Alignments 8+16+64 [FieldOffset(PayloadStart)] public Payload Value; From 89bd6552ffb40b2b7315e024ab1179636ed26962 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 02:46:06 +0100 Subject: [PATCH 25/42] lol; don't do extra work --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 003d4fda942..13c0a1a8ffa 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -112,6 +112,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 // Release the lock, input.Length is always positive so setting it is enough. Volatile.Write(ref e.LockAndLength, input.Length); } + return; Uncommon: if (input.Length == 0) From 42ef8f66b5018f30c7b72184ebbacfa437d22979 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 02:57:36 +0100 Subject: [PATCH 26/42] Faster compares --- src/Nethermind/Nethermind.Core/Address.cs | 8 +++++++- src/Nethermind/Nethermind.Core/StorageCell.cs | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Address.cs b/src/Nethermind/Nethermind.Core/Address.cs index 875dd2d17ae..4ec161024c2 100644 --- a/src/Nethermind/Nethermind.Core/Address.cs +++ b/src/Nethermind/Nethermind.Core/Address.cs @@ -7,6 +7,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Text.Json.Serialization; using Nethermind.Core.Crypto; @@ -148,7 +149,12 @@ public bool Equals(Address? other) return true; } - return Nethermind.Core.Extensions.Bytes.AreEqual(Bytes, other.Bytes); + ref byte bytes0 = ref MemoryMarshal.GetArrayDataReference(Bytes); + ref byte bytes1 = ref MemoryMarshal.GetArrayDataReference(other.Bytes); + // 20 bytes which is uint+Vector128 + return Unsafe.As(ref bytes0) == Unsafe.As(ref bytes1) && + Unsafe.As>(ref Unsafe.Add(ref bytes0, sizeof(uint))) == + Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint))); } public static Address FromNumber(in UInt256 number) diff --git a/src/Nethermind/Nethermind.Core/StorageCell.cs b/src/Nethermind/Nethermind.Core/StorageCell.cs index 47d6451ec8f..a35bc2eda97 100644 --- a/src/Nethermind/Nethermind.Core/StorageCell.cs +++ b/src/Nethermind/Nethermind.Core/StorageCell.cs @@ -47,7 +47,7 @@ public StorageCell(Address address, ValueHash256 hash) public bool Equals(StorageCell other) => _isHash == other._isHash && Unsafe.As>(ref Unsafe.AsRef(in _index)) == Unsafe.As>(ref Unsafe.AsRef(in other._index)) && - Address.Equals(other.Address) && Index.Equals(other.Index); + Address.Equals(other.Address); public override bool Equals(object? obj) { From 773ef2ac4965071ff792fcdea2a857e44ad7ad4f Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 04:53:39 +0100 Subject: [PATCH 27/42] Move other HasCodes to FastHash --- src/Nethermind/Nethermind.Core/Bloom.cs | 10 +---- .../Nethermind.Core/Extensions/Bytes.cs | 38 ++----------------- .../Nethermind.Serialization.Rlp/Rlp.cs | 5 +-- 3 files changed, 6 insertions(+), 47 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Bloom.cs b/src/Nethermind/Nethermind.Core/Bloom.cs index f980e1b4a0e..05d3f4e5d9f 100644 --- a/src/Nethermind/Nethermind.Core/Bloom.cs +++ b/src/Nethermind/Nethermind.Core/Bloom.cs @@ -114,10 +114,7 @@ public override bool Equals(object? obj) return Equals((Bloom)obj); } - public override int GetHashCode() - { - return Bytes.GetSimplifiedHashCode(); - } + public override int GetHashCode() => new ReadOnlySpan(Bytes).FastHash(); public void Add(LogEntry[] logEntries, Bloom? blockBloom = null) { @@ -278,10 +275,7 @@ public override readonly bool Equals(object? obj) return Equals((Bloom)obj); } - public override readonly int GetHashCode() - { - return Core.Extensions.Bytes.GetSimplifiedHashCode(Bytes); - } + public override readonly int GetHashCode() => Bytes.FastHash(); public readonly bool Matches(LogEntry logEntry) { diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 219adaaf2dc..a6bac0de85c 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -37,10 +37,7 @@ public override bool Equals(byte[]? x, byte[]? y) return AreEqual(x, y); } - public override int GetHashCode(byte[] obj) - { - return obj.GetSimplifiedHashCode(); - } + public override int GetHashCode(byte[] obj) => new ReadOnlySpan(obj).FastHash(); } private class NullableBytesEqualityComparer : EqualityComparer @@ -50,17 +47,14 @@ public override bool Equals(byte[]? x, byte[]? y) return AreEqual(x, y); } - public override int GetHashCode(byte[]? obj) - { - return obj?.GetSimplifiedHashCode() ?? 0; - } + public override int GetHashCode(byte[]? obj) => new ReadOnlySpan(obj).FastHash(); } private class SpanBytesEqualityComparer : ISpanEqualityComparer { public bool Equals(ReadOnlySpan x, ReadOnlySpan y) => AreEqual(x, y); - public int GetHashCode(ReadOnlySpan obj) => GetSimplifiedHashCode(obj); + public int GetHashCode(ReadOnlySpan obj) => obj.FastHash(); } public class BytesComparer : Comparer @@ -1122,32 +1116,6 @@ private static byte[] FromHexString(ReadOnlySpan hexString) return isSuccess ? result : throw new FormatException("Incorrect hex string"); } - [SuppressMessage("ReSharper", "NonReadonlyMemberInGetHashCode")] - public static int GetSimplifiedHashCode(this byte[] bytes) - { - const int fnvPrime = 0x01000193; - - if (bytes.Length == 0) - { - return 0; - } - - return (fnvPrime * bytes.Length * (((fnvPrime * (bytes[0] + 7)) ^ (bytes[^1] + 23)) + 11)) ^ (bytes[(bytes.Length - 1) / 2] + 53); - } - - [SuppressMessage("ReSharper", "NonReadonlyMemberInGetHashCode")] - public static int GetSimplifiedHashCode(this ReadOnlySpan bytes) - { - const int fnvPrime = 0x01000193; - - if (bytes.Length == 0) - { - return 0; - } - - return (fnvPrime * bytes.Length * (((fnvPrime * (bytes[0] + 7)) ^ (bytes[^1] + 23)) + 11)) ^ (bytes[(bytes.Length - 1) / 2] + 53); - } - public static void ChangeEndianness8(Span bytes) { if (bytes.Length % 16 != 0) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs index f6d97a79bf1..15642a95e09 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs @@ -1609,10 +1609,7 @@ public override bool Equals(object? other) return Equals(other as Rlp); } - public override int GetHashCode() - { - return Bytes is not null ? Bytes.GetSimplifiedHashCode() : 0; - } + public override int GetHashCode() => new ReadOnlySpan(Bytes).FastHash(); public bool Equals(Rlp? other) { From 226be3f0fc0d5990197c59aa67d598ccae57c9ec Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 05:15:45 +0100 Subject: [PATCH 28/42] Special case 32 and 20 bytes hashes --- .../Nethermind.Core/Crypto/KeccakCache.cs | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 13c0a1a8ffa..6d9df97efee 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Threading; using Nethermind.Core.Extensions; @@ -87,8 +88,33 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 Volatile.Write(ref e.LockAndLength, lockAndLength); // Lengths are equal, the input length can be used without any additional operation. - if (MemoryMarshal.CreateReadOnlySpan(ref copy.Start, input.Length).SequenceEqual(input)) + if (input.Length == 32) { + // Hashing UInt256 or Hash256 which is Vector256 + if (Unsafe.As>(ref copy.Start) == + Unsafe.As>(ref MemoryMarshal.GetReference(input))) + { + // Current keccak256 is correct hash. + return; + } + } + else if (input.Length == 20) + { + // Hashing Address + ref byte bytes0 = ref copy.Start; + ref byte bytes1 = ref MemoryMarshal.GetReference(input); + // 20 bytes which is uint+Vector128 + if (Unsafe.As(ref bytes0) == Unsafe.As(ref bytes1) && + Unsafe.As>(ref Unsafe.Add(ref bytes0, sizeof(uint))) == + Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint)))) + { + // Current keccak256 is correct hash. + return; + } + } + else if (MemoryMarshal.CreateReadOnlySpan(ref copy.Start, input.Length).SequenceEqual(input)) + { + // Non 32 byte or 20 byte input; call SequenceEqual. // Current keccak256 is correct hash. return; } @@ -107,7 +133,28 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 e.HashCode = hashCode; e.Keccak256 = keccak256; - input.CopyTo(MemoryMarshal.CreateSpan(ref e.Value.Start, input.Length)); + // Fast copy for 2 common sizes + if (input.Length == 32) + { + // UInt256 or Hash256 which is Vector256 + Unsafe.As>(ref e.Value.Start) = + Unsafe.As>(ref MemoryMarshal.GetReference(input)); + } + else if (input.Length == 20) + { + // Address + ref byte bytes0 = ref e.Value.Start; + ref byte bytes1 = ref MemoryMarshal.GetReference(input); + // 20 bytes which is uint+Vector128 + Unsafe.As(ref bytes0) = Unsafe.As(ref bytes1); + Unsafe.As>(ref Unsafe.Add(ref bytes0, sizeof(uint))) = + Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint))); + } + else + { + // Non 32 byte or 20 byte input; call CopyTo + input.CopyTo(MemoryMarshal.CreateSpan(ref e.Value.Start, input.Length)); + } // Release the lock, input.Length is always positive so setting it is enough. Volatile.Write(ref e.LockAndLength, input.Length); From 7d5fba51431c6652fbd2ce2da28be5380545d2d4 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 05:36:47 +0100 Subject: [PATCH 29/42] Word align vector compare --- .../Nethermind.Core/Crypto/KeccakCache.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 6d9df97efee..b6bb82b0b80 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -103,10 +103,10 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 // Hashing Address ref byte bytes0 = ref copy.Start; ref byte bytes1 = ref MemoryMarshal.GetReference(input); - // 20 bytes which is uint+Vector128 - if (Unsafe.As(ref bytes0) == Unsafe.As(ref bytes1) && - Unsafe.As>(ref Unsafe.Add(ref bytes0, sizeof(uint))) == - Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint)))) + // 20 bytes which is Vector128+uint + if (Unsafe.As>(ref bytes0) == Unsafe.As>(ref bytes1) && + Unsafe.As(ref Unsafe.Add(ref bytes0, Vector128.Count)) == + Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count))) { // Current keccak256 is correct hash. return; @@ -145,10 +145,10 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 // Address ref byte bytes0 = ref e.Value.Start; ref byte bytes1 = ref MemoryMarshal.GetReference(input); - // 20 bytes which is uint+Vector128 - Unsafe.As(ref bytes0) = Unsafe.As(ref bytes1); - Unsafe.As>(ref Unsafe.Add(ref bytes0, sizeof(uint))) = - Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint))); + // 20 bytes which is Vector128+uint + Unsafe.As>(ref bytes0) = Unsafe.As>(ref bytes1); + Unsafe.As(ref Unsafe.Add(ref bytes0, Vector128.Count)) = + Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count)); } else { From 7184d268ac0589ea9058885d30dac14987573d83 Mon Sep 17 00:00:00 2001 From: scooletz Date: Mon, 19 Aug 2024 11:25:11 +0200 Subject: [PATCH 30/42] constants --- .../Nethermind.Core/Crypto/KeccakCache.cs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index b6bb82b0b80..2af062a5097 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -17,8 +17,8 @@ namespace Nethermind.Core.Crypto; /// It allocates only 8MB of memory to store 64k of entries. /// No misaligned reads. Everything is aligned to both cache lines as well as to boundaries so no torn reads. /// Requires a single CAS to lock and to unlock. -/// On lock failure, it just moves on with execution. -/// Uses copying on the stack to get the entry, have it copied and release the lock ASAP. This is 128 bytes to copy that quite likely will be the hit. +/// On a lock failure, it just moves on with execution. +/// When a potential hit happens, the value of the result and the value of the key are copied on the stack to release the lock ASAP. /// public static unsafe class KeccakCache { @@ -28,6 +28,9 @@ public static unsafe class KeccakCache public const int Count = BucketMask + 1; private const int BucketMask = 0x0000_FFFF; + private const int InputLengthOfKeccak = ValueHash256.MemorySize; + private const int InputLengthOfAddress = Address.Size; + private static readonly Entry* Memory; static KeccakCache() @@ -88,7 +91,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 Volatile.Write(ref e.LockAndLength, lockAndLength); // Lengths are equal, the input length can be used without any additional operation. - if (input.Length == 32) + if (input.Length == InputLengthOfKeccak) { // Hashing UInt256 or Hash256 which is Vector256 if (Unsafe.As>(ref copy.Start) == @@ -98,7 +101,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 return; } } - else if (input.Length == 20) + else if (input.Length == InputLengthOfAddress) { // Hashing Address ref byte bytes0 = ref copy.Start; @@ -134,13 +137,13 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 e.Keccak256 = keccak256; // Fast copy for 2 common sizes - if (input.Length == 32) + if (input.Length == InputLengthOfKeccak) { // UInt256 or Hash256 which is Vector256 Unsafe.As>(ref e.Value.Start) = Unsafe.As>(ref MemoryMarshal.GetReference(input)); } - else if (input.Length == 20) + else if (input.Length == InputLengthOfAddress) { // Address ref byte bytes0 = ref e.Value.Start; From 795b4c93d768daac4b67134f5414e315240672f0 Mon Sep 17 00:00:00 2001 From: scooletz Date: Mon, 19 Aug 2024 13:02:06 +0200 Subject: [PATCH 31/42] stack reduced by 8 and one less comparison on read --- .../Nethermind.Core/Crypto/KeccakCache.cs | 118 ++++++++---------- 1 file changed, 53 insertions(+), 65 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 2af062a5097..515f2e40c28 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -26,7 +26,10 @@ public static unsafe class KeccakCache /// Count is defined as a +1 over bucket mask. In the future, just change the mask as the main parameter. /// public const int Count = BucketMask + 1; + private const int BucketMask = 0x0000_FFFF; + private const uint HashMask = 0xFFFF_0000; + private const uint LockMarker = 0x0000_8000; private const int InputLengthOfKeccak = ValueHash256.MemorySize; private const int InputLengthOfAddress = Address.Size; @@ -65,75 +68,68 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef(Memory), index); - // Read aligned, volatile, won't be torn, check with computed - if (Volatile.Read(ref e.HashCode) == hashCode) + uint combined = (HashMask & (uint)hashCode) | (uint)input.Length; + + // Check combined. It's aligned properly and can be read in one go. + uint existing = Volatile.Read(ref e.Combined); + + // Lock by negating length if the length is the same size as input. + if (existing == combined && + Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) { - // There's a possibility of a hit, try lock. - int lockAndLength = Volatile.Read(ref e.LockAndLength); - // Lock by negating length if the length is the same size as input. - if (lockAndLength == input.Length && Interlocked.CompareExchange(ref e.LockAndLength, -lockAndLength, lockAndLength) == lockAndLength) - { - if (e.HashCode != hashCode) - { - // The value has been changed between reading and taking a lock. - // Release the lock and compute. - Volatile.Write(ref e.LockAndLength, lockAndLength); - goto Compute; - } + // Combined is equal to existing, meaning that it was not locked, and both the length and the hash were equal. - // Take local copy of the payload and hash, to release the lock as soon as possible and make a key comparison without holding it. - // Local copy of 8+16+64 payload bytes. - Payload copy = e.Value; - // Copy Keccak256 directly to the local return variable, since we will overwrite if no match anyway. - keccak256 = e.Keccak256; + // Take local copy of the payload and hash, to release the lock as soon as possible and make a key comparison without holding it. + // Local copy of 8+16+64 payload bytes. + Payload copy = e.Value; + // Copy Keccak256 directly to the local return variable, since we will overwrite if no match anyway. + keccak256 = e.Keccak256; - // Release the lock, by setting back to unnegated length. - Volatile.Write(ref e.LockAndLength, lockAndLength); + // Release the lock, by setting back to unnegated length. + Volatile.Write(ref e.Combined, combined); - // Lengths are equal, the input length can be used without any additional operation. - if (input.Length == InputLengthOfKeccak) - { - // Hashing UInt256 or Hash256 which is Vector256 - if (Unsafe.As>(ref copy.Start) == - Unsafe.As>(ref MemoryMarshal.GetReference(input))) - { - // Current keccak256 is correct hash. - return; - } - } - else if (input.Length == InputLengthOfAddress) + // Lengths are equal, the input length can be used without any additional operation. + if (input.Length == InputLengthOfKeccak) + { + // Hashing UInt256 or Hash256 which is Vector256 + if (Unsafe.As>(ref copy.Start) == + Unsafe.As>(ref MemoryMarshal.GetReference(input))) { - // Hashing Address - ref byte bytes0 = ref copy.Start; - ref byte bytes1 = ref MemoryMarshal.GetReference(input); - // 20 bytes which is Vector128+uint - if (Unsafe.As>(ref bytes0) == Unsafe.As>(ref bytes1) && - Unsafe.As(ref Unsafe.Add(ref bytes0, Vector128.Count)) == - Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count))) - { - // Current keccak256 is correct hash. - return; - } + // Current keccak256 is correct hash. + return; } - else if (MemoryMarshal.CreateReadOnlySpan(ref copy.Start, input.Length).SequenceEqual(input)) + } + else if (input.Length == InputLengthOfAddress) + { + // Hashing Address + ref byte bytes0 = ref copy.Start; + ref byte bytes1 = ref MemoryMarshal.GetReference(input); + // 20 bytes which is Vector128+uint + if (Unsafe.As>(ref bytes0) == Unsafe.As>(ref bytes1) && + Unsafe.As(ref Unsafe.Add(ref bytes0, Vector128.Count)) == + Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count))) { - // Non 32 byte or 20 byte input; call SequenceEqual. // Current keccak256 is correct hash. return; } } + else if (MemoryMarshal.CreateReadOnlySpan(ref copy.Start, input.Length).SequenceEqual(input)) + { + // Non 32 byte or 20 byte input; call SequenceEqual. + // Current keccak256 is correct hash. + return; + } } - Compute: keccak256 = ValueKeccak.Compute(input); // Try lock and memoize - int length = Volatile.Read(ref e.LockAndLength); + existing = Volatile.Read(ref e.Combined); + // Negative value means that the entry is locked, set to int.MinValue to avoid confusion empty entry, // since we are overwriting it anyway e.g. -0 would not be a reliable locked state. - if (length >= 0 && Interlocked.CompareExchange(ref e.LockAndLength, int.MinValue, length) == length) + if ((existing & LockMarker) == 0 && Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) { - e.HashCode = hashCode; e.Keccak256 = keccak256; // Fast copy for 2 common sizes @@ -160,11 +156,12 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 } // Release the lock, input.Length is always positive so setting it is enough. - Volatile.Write(ref e.LockAndLength, input.Length); + Volatile.Write(ref e.Combined, combined); } + return; - Uncommon: + Uncommon: if (input.Length == 0) { keccak256 = ValueKeccak.OfAnEmptyString; @@ -196,29 +193,20 @@ private struct Entry public const int MaxPayloadLength = ValueStart - PayloadStart; /// - /// Length is always positive so we can use a negative length to indicate that it is locked. - /// - [FieldOffset(0)] - public int LockAndLength; - - /// - /// The fast Crc32c of the Value + /// Represents a combined value for: hash, length and a potential . /// - [FieldOffset(4)] - public int HashCode; + [FieldOffset(0)] public uint Combined; /// /// The actual value /// /// Alignments 8+16+64 - [FieldOffset(PayloadStart)] - public Payload Value; + [FieldOffset(PayloadStart)] public Payload Value; /// /// The Keccak of the Value /// - [FieldOffset(ValueStart)] - public ValueHash256 Keccak256; + [FieldOffset(ValueStart)] public ValueHash256 Keccak256; } [InlineArray(Entry.MaxPayloadLength)] From c19edb33debb6615a6115726fe7000dd54989a63 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 12:23:57 +0100 Subject: [PATCH 32/42] Use full entropy of HashCode for comparision --- .../Nethermind.Core/Crypto/KeccakCache.cs | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 515f2e40c28..df39af26f71 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -25,11 +25,8 @@ public static unsafe class KeccakCache /// /// Count is defined as a +1 over bucket mask. In the future, just change the mask as the main parameter. /// - public const int Count = BucketMask + 1; - - private const int BucketMask = 0x0000_FFFF; - private const uint HashMask = 0xFFFF_0000; - private const uint LockMarker = 0x0000_8000; + private const uint BucketMask = 0x0000_FFFF; + public const uint Count = BucketMask + 1; private const int InputLengthOfKeccak = ValueHash256.MemorySize; private const int InputLengthOfAddress = Address.Size; @@ -66,19 +63,16 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 Debug.Assert(index < Count); + // Have to cast hashCode via uint to avoid sign extension into the long. + long combined = (long)input.Length << 32 | (long)(uint)hashCode; ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef(Memory), index); - uint combined = (HashMask & (uint)hashCode) | (uint)input.Length; - - // Check combined. It's aligned properly and can be read in one go. - uint existing = Volatile.Read(ref e.Combined); - - // Lock by negating length if the length is the same size as input. - if (existing == combined && - Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) - { + // Read aligned, volatile, won't be torn, check with computed + if (Volatile.Read(ref e.LockAndLengthAndHash) == combined && // Combined is equal to existing, meaning that it was not locked, and both the length and the hash were equal. - + // Lock by negating combined, since the length is always positive, negative always indicates locked. + Interlocked.CompareExchange(ref e.LockAndLengthAndHash, -combined, combined) == combined) + { // Take local copy of the payload and hash, to release the lock as soon as possible and make a key comparison without holding it. // Local copy of 8+16+64 payload bytes. Payload copy = e.Value; @@ -86,7 +80,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 keccak256 = e.Keccak256; // Release the lock, by setting back to unnegated length. - Volatile.Write(ref e.Combined, combined); + Volatile.Write(ref e.LockAndLengthAndHash, combined); // Lengths are equal, the input length can be used without any additional operation. if (input.Length == InputLengthOfKeccak) @@ -107,7 +101,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 // 20 bytes which is Vector128+uint if (Unsafe.As>(ref bytes0) == Unsafe.As>(ref bytes1) && Unsafe.As(ref Unsafe.Add(ref bytes0, Vector128.Count)) == - Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count))) + Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count))) { // Current keccak256 is correct hash. return; @@ -124,11 +118,10 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 keccak256 = ValueKeccak.Compute(input); // Try lock and memoize - existing = Volatile.Read(ref e.Combined); - - // Negative value means that the entry is locked, set to int.MinValue to avoid confusion empty entry, + long current = Volatile.Read(ref e.LockAndLengthAndHash); + // Negative value means that the entry is locked, set to long.MinValue to avoid confusion empty entry, // since we are overwriting it anyway e.g. -0 would not be a reliable locked state. - if ((existing & LockMarker) == 0 && Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) + if (current >= 0 && Interlocked.CompareExchange(ref e.LockAndLengthAndHash, -combined, current) == current) { e.Keccak256 = keccak256; @@ -156,12 +149,11 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 } // Release the lock, input.Length is always positive so setting it is enough. - Volatile.Write(ref e.Combined, combined); + Volatile.Write(ref e.LockAndLengthAndHash, combined); } - return; - Uncommon: + Uncommon: if (input.Length == 0) { keccak256 = ValueKeccak.OfAnEmptyString; @@ -188,14 +180,14 @@ private struct Entry /// public const int Size = 128; - private const int PayloadStart = 8; + private const int PayloadStart = sizeof(ulong); private const int ValueStart = Size - ValueHash256.MemorySize; public const int MaxPayloadLength = ValueStart - PayloadStart; /// - /// Represents a combined value for: hash, length and a potential . + /// Length is always positive so we can use a negative length to indicate that it is locked. /// - [FieldOffset(0)] public uint Combined; + [FieldOffset(0)] public long LockAndLengthAndHash; /// /// The actual value From 1f8b94fc0a1bac27f05964c2c64d1384c77aed34 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 12:29:59 +0100 Subject: [PATCH 33/42] Revert "Use full entropy of HashCode for comparision" This reverts commit c19edb33debb6615a6115726fe7000dd54989a63. --- .../Nethermind.Core/Crypto/KeccakCache.cs | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index df39af26f71..515f2e40c28 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -25,8 +25,11 @@ public static unsafe class KeccakCache /// /// Count is defined as a +1 over bucket mask. In the future, just change the mask as the main parameter. /// - private const uint BucketMask = 0x0000_FFFF; - public const uint Count = BucketMask + 1; + public const int Count = BucketMask + 1; + + private const int BucketMask = 0x0000_FFFF; + private const uint HashMask = 0xFFFF_0000; + private const uint LockMarker = 0x0000_8000; private const int InputLengthOfKeccak = ValueHash256.MemorySize; private const int InputLengthOfAddress = Address.Size; @@ -63,16 +66,19 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 Debug.Assert(index < Count); - // Have to cast hashCode via uint to avoid sign extension into the long. - long combined = (long)input.Length << 32 | (long)(uint)hashCode; ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef(Memory), index); - // Read aligned, volatile, won't be torn, check with computed - if (Volatile.Read(ref e.LockAndLengthAndHash) == combined && - // Combined is equal to existing, meaning that it was not locked, and both the length and the hash were equal. - // Lock by negating combined, since the length is always positive, negative always indicates locked. - Interlocked.CompareExchange(ref e.LockAndLengthAndHash, -combined, combined) == combined) + uint combined = (HashMask & (uint)hashCode) | (uint)input.Length; + + // Check combined. It's aligned properly and can be read in one go. + uint existing = Volatile.Read(ref e.Combined); + + // Lock by negating length if the length is the same size as input. + if (existing == combined && + Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) { + // Combined is equal to existing, meaning that it was not locked, and both the length and the hash were equal. + // Take local copy of the payload and hash, to release the lock as soon as possible and make a key comparison without holding it. // Local copy of 8+16+64 payload bytes. Payload copy = e.Value; @@ -80,7 +86,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 keccak256 = e.Keccak256; // Release the lock, by setting back to unnegated length. - Volatile.Write(ref e.LockAndLengthAndHash, combined); + Volatile.Write(ref e.Combined, combined); // Lengths are equal, the input length can be used without any additional operation. if (input.Length == InputLengthOfKeccak) @@ -101,7 +107,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 // 20 bytes which is Vector128+uint if (Unsafe.As>(ref bytes0) == Unsafe.As>(ref bytes1) && Unsafe.As(ref Unsafe.Add(ref bytes0, Vector128.Count)) == - Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count))) + Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count))) { // Current keccak256 is correct hash. return; @@ -118,10 +124,11 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 keccak256 = ValueKeccak.Compute(input); // Try lock and memoize - long current = Volatile.Read(ref e.LockAndLengthAndHash); - // Negative value means that the entry is locked, set to long.MinValue to avoid confusion empty entry, + existing = Volatile.Read(ref e.Combined); + + // Negative value means that the entry is locked, set to int.MinValue to avoid confusion empty entry, // since we are overwriting it anyway e.g. -0 would not be a reliable locked state. - if (current >= 0 && Interlocked.CompareExchange(ref e.LockAndLengthAndHash, -combined, current) == current) + if ((existing & LockMarker) == 0 && Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) { e.Keccak256 = keccak256; @@ -149,11 +156,12 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 } // Release the lock, input.Length is always positive so setting it is enough. - Volatile.Write(ref e.LockAndLengthAndHash, combined); + Volatile.Write(ref e.Combined, combined); } + return; - Uncommon: + Uncommon: if (input.Length == 0) { keccak256 = ValueKeccak.OfAnEmptyString; @@ -180,14 +188,14 @@ private struct Entry /// public const int Size = 128; - private const int PayloadStart = sizeof(ulong); + private const int PayloadStart = 8; private const int ValueStart = Size - ValueHash256.MemorySize; public const int MaxPayloadLength = ValueStart - PayloadStart; /// - /// Length is always positive so we can use a negative length to indicate that it is locked. + /// Represents a combined value for: hash, length and a potential . /// - [FieldOffset(0)] public long LockAndLengthAndHash; + [FieldOffset(0)] public uint Combined; /// /// The actual value From 928929e20352b1d79107c9fd59b612411790bbcc Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 12:41:51 +0100 Subject: [PATCH 34/42] Tweaks --- .../Nethermind.Core/Crypto/KeccakCache.cs | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 515f2e40c28..ef5d7c6cab0 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -67,15 +67,14 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 Debug.Assert(index < Count); ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef(Memory), index); - + // Half the hash his encoded in the bucket so we only need half of it and can use other half for length. uint combined = (HashMask & (uint)hashCode) | (uint)input.Length; // Check combined. It's aligned properly and can be read in one go. - uint existing = Volatile.Read(ref e.Combined); // Lock by negating length if the length is the same size as input. - if (existing == combined && - Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) + if (Volatile.Read(ref e.Combined) == combined && + Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, combined) == combined) { // Combined is equal to existing, meaning that it was not locked, and both the length and the hash were equal. @@ -104,10 +103,10 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 // Hashing Address ref byte bytes0 = ref copy.Start; ref byte bytes1 = ref MemoryMarshal.GetReference(input); - // 20 bytes which is Vector128+uint - if (Unsafe.As>(ref bytes0) == Unsafe.As>(ref bytes1) && - Unsafe.As(ref Unsafe.Add(ref bytes0, Vector128.Count)) == - Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count))) + // 20 bytes which is uint+Vector128 + if (Unsafe.As(ref bytes0) == Unsafe.As(ref bytes1) && + Unsafe.As>(ref Unsafe.Add(ref bytes0, sizeof(uint))) == + Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint)))) { // Current keccak256 is correct hash. return; @@ -124,10 +123,9 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 keccak256 = ValueKeccak.Compute(input); // Try lock and memoize - existing = Volatile.Read(ref e.Combined); + uint existing = Volatile.Read(ref e.Combined); - // Negative value means that the entry is locked, set to int.MinValue to avoid confusion empty entry, - // since we are overwriting it anyway e.g. -0 would not be a reliable locked state. + // Set to locked to combined locked state, if not already locked. if ((existing & LockMarker) == 0 && Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) { e.Keccak256 = keccak256; @@ -144,10 +142,10 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 // Address ref byte bytes0 = ref e.Value.Start; ref byte bytes1 = ref MemoryMarshal.GetReference(input); - // 20 bytes which is Vector128+uint - Unsafe.As>(ref bytes0) = Unsafe.As>(ref bytes1); - Unsafe.As(ref Unsafe.Add(ref bytes0, Vector128.Count)) = - Unsafe.As(ref Unsafe.Add(ref bytes1, Vector128.Count)); + // 20 bytes which is uint+Vector128 + Unsafe.As(ref bytes0) = Unsafe.As(ref bytes1); + Unsafe.As>(ref Unsafe.Add(ref bytes0, sizeof(uint))) = + Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint))); } else { @@ -161,7 +159,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 return; - Uncommon: + Uncommon: if (input.Length == 0) { keccak256 = ValueKeccak.OfAnEmptyString; @@ -188,7 +186,7 @@ private struct Entry /// public const int Size = 128; - private const int PayloadStart = 8; + private const int PayloadStart = sizeof(uint); private const int ValueStart = Size - ValueHash256.MemorySize; public const int MaxPayloadLength = ValueStart - PayloadStart; From e32f995649a499a08deb6d7219449bc0f69baf20 Mon Sep 17 00:00:00 2001 From: scooletz Date: Mon, 19 Aug 2024 14:16:07 +0200 Subject: [PATCH 35/42] comments --- .../Nethermind.Core/Crypto/KeccakCache.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index ef5d7c6cab0..980e4513fae 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -67,12 +67,12 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 Debug.Assert(index < Count); ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef(Memory), index); + // Half the hash his encoded in the bucket so we only need half of it and can use other half for length. + // This allows to create a combined value that represents a part of the hash, the input's length and the lock marker. uint combined = (HashMask & (uint)hashCode) | (uint)input.Length; - // Check combined. It's aligned properly and can be read in one go. - - // Lock by negating length if the length is the same size as input. + // Compare with volatile read and then try to lock with CAS if (Volatile.Read(ref e.Combined) == combined && Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, combined) == combined) { @@ -84,7 +84,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 // Copy Keccak256 directly to the local return variable, since we will overwrite if no match anyway. keccak256 = e.Keccak256; - // Release the lock, by setting back to unnegated length. + // Release the lock, by setting back to the combined value. Volatile.Write(ref e.Combined, combined); // Lengths are equal, the input length can be used without any additional operation. @@ -122,10 +122,9 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 keccak256 = ValueKeccak.Compute(input); - // Try lock and memoize uint existing = Volatile.Read(ref e.Combined); - // Set to locked to combined locked state, if not already locked. + // Try to set to the combined locked state, if not already locked. if ((existing & LockMarker) == 0 && Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) { e.Keccak256 = keccak256; @@ -153,7 +152,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 input.CopyTo(MemoryMarshal.CreateSpan(ref e.Value.Start, input.Length)); } - // Release the lock, input.Length is always positive so setting it is enough. + // Release the lock, by setting back to the combined value. Volatile.Write(ref e.Combined, combined); } From 6938aab0ee2f8b6f850cc44a7bd20068ccaa3c71 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 19 Aug 2024 13:22:02 +0100 Subject: [PATCH 36/42] Align 32 --- .../Nethermind.Core/Crypto/KeccakCache.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index ef5d7c6cab0..0a0332b82ff 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -91,7 +91,7 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 if (input.Length == InputLengthOfKeccak) { // Hashing UInt256 or Hash256 which is Vector256 - if (Unsafe.As>(ref copy.Start) == + if (Unsafe.As>(ref copy.Aligned32) == Unsafe.As>(ref MemoryMarshal.GetReference(input))) { // Current keccak256 is correct hash. @@ -101,11 +101,10 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 else if (input.Length == InputLengthOfAddress) { // Hashing Address - ref byte bytes0 = ref copy.Start; ref byte bytes1 = ref MemoryMarshal.GetReference(input); // 20 bytes which is uint+Vector128 - if (Unsafe.As(ref bytes0) == Unsafe.As(ref bytes1) && - Unsafe.As>(ref Unsafe.Add(ref bytes0, sizeof(uint))) == + if (Unsafe.As(ref copy.Start) == Unsafe.As(ref bytes1) && + Unsafe.As>(ref copy.Aligned32) == Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint)))) { // Current keccak256 is correct hash. @@ -134,17 +133,16 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 if (input.Length == InputLengthOfKeccak) { // UInt256 or Hash256 which is Vector256 - Unsafe.As>(ref e.Value.Start) = + Unsafe.As>(ref e.Value.Aligned32) = Unsafe.As>(ref MemoryMarshal.GetReference(input)); } else if (input.Length == InputLengthOfAddress) { // Address - ref byte bytes0 = ref e.Value.Start; ref byte bytes1 = ref MemoryMarshal.GetReference(input); // 20 bytes which is uint+Vector128 - Unsafe.As(ref bytes0) = Unsafe.As(ref bytes1); - Unsafe.As>(ref Unsafe.Add(ref bytes0, sizeof(uint))) = + Unsafe.As(ref e.Value.Start) = Unsafe.As(ref bytes1); + Unsafe.As>(ref e.Value.Aligned32) = Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint))); } else @@ -198,18 +196,20 @@ private struct Entry /// /// The actual value /// - /// Alignments 8+16+64 + /// Alignments 4+8+16+64 [FieldOffset(PayloadStart)] public Payload Value; - /// /// The Keccak of the Value /// [FieldOffset(ValueStart)] public ValueHash256 Keccak256; } - [InlineArray(Entry.MaxPayloadLength)] + [StructLayout(LayoutKind.Explicit, Size = Entry.MaxPayloadLength)] private struct Payload { - public byte Start; + private const int AlignedStart = Entry.MaxPayloadLength - 32; + + [FieldOffset(0)] public byte Start; + [FieldOffset(AlignedStart)] public byte Aligned32; } } From 472ca2677f781d6a98685f87eaa19750f1563f8a Mon Sep 17 00:00:00 2001 From: scooletz Date: Tue, 20 Aug 2024 10:11:59 +0200 Subject: [PATCH 37/42] smaller entry, bigger cache --- .../Nethermind.Core.Test/KeccakCacheTests.cs | 33 ++++++++++++++++--- .../Nethermind.Core/Crypto/KeccakCache.cs | 16 +++++---- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs index ea676bce25f..8e95d13de24 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs @@ -47,15 +47,40 @@ public void Very_long() KeccakCache.Compute(span).Should().Be(ValueKeccak.Compute(span)); } + [Test] + [Explicit("Used to create collisions")] + public void Print_collisions() + { + var random = new Random(13); + Span span = stackalloc byte[32]; + + random.NextBytes(span); + var bucket = KeccakCache.GetBucket(span); + + Console.WriteLine(span.ToHexString()); + + var found = 1; + + while (found < 4) + { + random.NextBytes(span); + if (KeccakCache.GetBucket(span) == bucket) + { + Console.WriteLine(span.ToHexString()); + found++; + } + } + } + [Test] public void Collision() { var colliding = new[] { - "f8ae910727f29363002d948385ff15bc6a9bacbef13bc2afc0aa8d02749668", - "baec2065df3da176cee21714b7bfb00d0c57f37a21daf2b2d4056f67270290", - "924cc47a10ad801c74a491b19492563d2351b285ff1679e5b5264e57b13bbb", - "4fb39f7800b3a43e4e722dc6fed03b126e0125d7ca713b0558564a29903ea9", + "50f78269ea2ddd2d6ab4338fd5c7909c229561e565f6b04b9447b0bd73585687", + "de75b3e495a58811469fb21345c7c1f84db0a3e1a3bf628c5689b53520af94de", + "82be999650f45409208eacb42f357695bca746f58fb35c0a4a4d09d5a2ac066a", + "f71034d862639845003bdc2d0d30ed1f8bd24c77573026fe9b838f33e72dcc6d", }; var collisions = colliding.Length; diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 441a4a77b71..63784b3f431 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -14,7 +15,7 @@ namespace Nethermind.Core.Crypto; /// /// This is a minimalistic one-way set associative cache for Keccak values. /// -/// It allocates only 8MB of memory to store 64k of entries. +/// It allocates only 12MB of memory to store 128k of entries. /// No misaligned reads. Everything is aligned to both cache lines as well as to boundaries so no torn reads. /// Requires a single CAS to lock and to unlock. /// On a lock failure, it just moves on with execution. @@ -27,8 +28,8 @@ public static unsafe class KeccakCache /// public const int Count = BucketMask + 1; - private const int BucketMask = 0x0000_FFFF; - private const uint HashMask = 0xFFFF_0000; + private const int BucketMask = 0x0001_FFFF; + private const uint HashMask = unchecked((uint)~BucketMask); private const uint LockMarker = 0x0000_8000; private const int InputLengthOfKeccak = ValueHash256.MemorySize; @@ -41,7 +42,7 @@ static KeccakCache() const UIntPtr size = Count * Entry.Size; // Aligned, so that no torn reads if fields of Entry are properly aligned. - Memory = (Entry*)NativeMemory.AlignedAlloc(size, Entry.Size); + Memory = (Entry*)NativeMemory.AlignedAlloc(size, BitOperations.RoundUpToPowerOf2(Entry.Size)); NativeMemory.Clear(Memory, size); } @@ -179,9 +180,10 @@ public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak25 private struct Entry { /// - /// Should work for both ARM and x64 and be aligned. + /// The size will make it 1.5 CPU cache entry or 0.75 which may result in some collisions. + /// Still, it's better to save these 32 bytes per entry and have a bigger cache. /// - public const int Size = 128; + public const int Size = 96; private const int PayloadStart = sizeof(uint); private const int ValueStart = Size - ValueHash256.MemorySize; @@ -195,8 +197,8 @@ private struct Entry /// /// The actual value /// - /// Alignments 4+8+16+64 [FieldOffset(PayloadStart)] public Payload Value; + /// /// The Keccak of the Value /// From 7d1bba699d536374bf3fca62d42214c26f7054ad Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 20 Aug 2024 18:31:51 +0100 Subject: [PATCH 38/42] Missed one --- src/Nethermind/Nethermind.Core/Address.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Core/Address.cs b/src/Nethermind/Nethermind.Core/Address.cs index 4ec161024c2..b4f86d6d8d0 100644 --- a/src/Nethermind/Nethermind.Core/Address.cs +++ b/src/Nethermind/Nethermind.Core/Address.cs @@ -228,7 +228,7 @@ public override bool CanConvertTo(ITypeDescriptorContext? context, Type? destina destinationType == typeof(string) || base.CanConvertTo(context, destinationType); } - public Hash256 ToAccountPath => Keccak.Compute(Bytes); + public Hash256 ToAccountPath => KeccakCache.Compute(Bytes); [SkipLocalsInit] public ValueHash256 ToHash() From 50c1e8f40892957cd924f63995d7aad1826be087 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 31 Aug 2024 12:16:44 +0100 Subject: [PATCH 39/42] Faster FastHash --- .../Nethermind.Core.Test/KeccakCacheTests.cs | 20 ++--- .../Extensions/SpanExtensions.cs | 75 ++++++++++++------- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs index 8e95d13de24..b2827495ba5 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs @@ -47,41 +47,41 @@ public void Very_long() KeccakCache.Compute(span).Should().Be(ValueKeccak.Compute(span)); } - [Test] - [Explicit("Used to create collisions")] - public void Print_collisions() + private string[] GetBucketCollisions() { var random = new Random(13); Span span = stackalloc byte[32]; + string[] collisions = new string[4]; random.NextBytes(span); var bucket = KeccakCache.GetBucket(span); Console.WriteLine(span.ToHexString()); + collisions[0] = span.ToHexString(); var found = 1; + ulong iterations = 0; while (found < 4) { random.NextBytes(span); if (KeccakCache.GetBucket(span) == bucket) { + collisions[found] = span.ToHexString(); Console.WriteLine(span.ToHexString()); found++; } + iterations++; } + + Console.WriteLine($"{iterations} iterations to find"); + return collisions; } [Test] public void Collision() { - var colliding = new[] - { - "50f78269ea2ddd2d6ab4338fd5c7909c229561e565f6b04b9447b0bd73585687", - "de75b3e495a58811469fb21345c7c1f84db0a3e1a3bf628c5689b53520af94de", - "82be999650f45409208eacb42f357695bca746f58fb35c0a4a4d09d5a2ac066a", - "f71034d862639845003bdc2d0d30ed1f8bd24c77573026fe9b838f33e72dcc6d", - }; + var colliding = GetBucketCollisions(); var collisions = colliding.Length; var array = colliding.Select(c => Bytes.FromHexString(c)).ToArray(); diff --git a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs index 35e58818ad2..816f3a7fa1d 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs @@ -176,48 +176,49 @@ public static ArrayPoolList ToPooledList(this in ReadOnlySpan span) [SkipLocalsInit] public static int FastHash(this ReadOnlySpan input) { + // Very fast hardware accelerated non-cryptographic hash function var length = input.Length; if (length == 0) return 0; ref var b = ref MemoryMarshal.GetReference(input); + uint hash = s_instanceRandom + (uint)length; + if (length < sizeof(long)) + { + goto Short; + } - // Start with instance random, length and first as seed - uint hash = (s_instanceRandom + (uint)length) ^ b; - - // This is done below, without branches - // if ((length & 1) == 1) - // { - // hash = b; - // b = ref Unsafe.Add(ref b, 1); - // length -= 1; - // } - - uint bit = (uint)length & sizeof(byte); - b = ref Unsafe.Add(ref b, bit); - length -= (int)bit; - - if ((length & sizeof(ushort)) != 0) + // Start with instance random, length and first ulong as seed + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + if ((length & sizeof(ulong)) != 0) { - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); - b = ref Unsafe.Add(ref b, sizeof(ushort)); - length -= sizeof(ushort); + if (length == sizeof(ulong)) + { + // Exactly ulong length return the hash + return (int)hash; + } + // Already Crc'd first ulong so skip it + b = ref Unsafe.Add(ref b, sizeof(ulong)); + length -= sizeof(ulong); } - if ((length & sizeof(uint)) != 0) + else { - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); - b = ref Unsafe.Add(ref b, sizeof(uint)); - length -= sizeof(uint); + // Skip the bytes that don't align to ulong + uint bits = (uint)length & 7; + b = ref Unsafe.Add(ref b, bits); + length -= (int)bits; } while (length >= sizeof(ulong) * 3) { - // Crc32C is 3 cycle latency, 1 cycle throughput. However is complex to fully - // break the dependency chain when most of our hashes are on <= 32 byte data. - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref b, sizeof(ulong)))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref b, sizeof(ulong) * 2))); + // Crc32C is 3 cycle latency, 1 cycle throughput + // So we us same initial 3 times to not create a dependency chain + uint hash0 = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + uint hash1 = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref b, sizeof(ulong)))); + uint hash2 = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref b, sizeof(ulong) * 2))); b = ref Unsafe.Add(ref b, sizeof(ulong) * 3); length -= sizeof(ulong) * 3; + // Combine the 3 hashes; performing the shift on first crc to calculate + hash = BitOperations.Crc32C(hash1, ((ulong)hash0 << (sizeof(uint) * 8)) | hash2); } while (length > 0) @@ -228,6 +229,24 @@ public static int FastHash(this ReadOnlySpan input) } return (int)hash; + Short: + ulong data = 0; + if ((length & sizeof(byte)) != 0) + { + data = b; + b = ref Unsafe.Add(ref b, sizeof(byte)); + } + if ((length & sizeof(ushort)) != 0) + { + data = (data << (sizeof(ushort) * 8)) | Unsafe.ReadUnaligned(ref b); + b = ref Unsafe.Add(ref b, sizeof(ushort)); + } + if ((length & sizeof(uint)) != 0) + { + data = (data << (sizeof(uint) * 8)) | Unsafe.ReadUnaligned(ref b); + } + + return (int)BitOperations.Crc32C(hash, data); } } } From fc52fcc9b40c198fbb0a003b0b1ef01e157467a2 Mon Sep 17 00:00:00 2001 From: scooletz Date: Mon, 2 Sep 2024 13:31:25 +0200 Subject: [PATCH 40/42] alignment handled with if --- .../Extensions/SpanExtensions.cs | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs index 816f3a7fa1d..1d6f49d57ee 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs @@ -189,25 +189,24 @@ public static int FastHash(this ReadOnlySpan input) // Start with instance random, length and first ulong as seed hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); - if ((length & sizeof(ulong)) != 0) + + // Calculate misalignment and move by it if needed. + // If no misalignment, advance by the size of ulong + uint misaligned = (uint)length & 7; + if (misaligned != 0) { - if (length == sizeof(ulong)) - { - // Exactly ulong length return the hash - return (int)hash; - } - // Already Crc'd first ulong so skip it - b = ref Unsafe.Add(ref b, sizeof(ulong)); - length -= sizeof(ulong); + // Align by moving by the misaligned count + b = ref Unsafe.Add(ref b, misaligned); + length -= (int)misaligned; } else { - // Skip the bytes that don't align to ulong - uint bits = (uint)length & 7; - b = ref Unsafe.Add(ref b, bits); - length -= (int)bits; + // Already Crc'd first ulong so skip it + b = ref Unsafe.Add(ref b, sizeof(ulong)); + length -= sizeof(ulong); } + // Length is fully aligned here and b is set in place while (length >= sizeof(ulong) * 3) { // Crc32C is 3 cycle latency, 1 cycle throughput From 7363f2c8cf46a1993b014fe7b9a6183cdd5f4774 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 19 Sep 2024 14:47:33 +0100 Subject: [PATCH 41/42] Missed one fastHash --- .../PersistentStorageProvider.cs | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs index 9469208c731..af65595b304 100644 --- a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs +++ b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs @@ -2,10 +2,12 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Core.Collections; @@ -38,7 +40,7 @@ internal sealed class PersistentStorageProvider : PartialStorageProviderBase private readonly Dictionary _originalValues = new(); private readonly HashSet _committedThisRound = new(); - private readonly Dictionary> _blockCache = new(4_096); + private readonly Dictionary> _blockCache = new(4_096); private readonly ConcurrentDictionary? _preBlockCache; private readonly Func _loadFromTree; @@ -303,10 +305,10 @@ private void SaveToTree(HashSet toUpdateRoots, Change change) toUpdateRoots.Add(change.StorageCell.Address); tree.Set(change.StorageCell.Index, change.Value); - ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, change.StorageCell.Address, out bool exists); + ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, change.StorageCell.Address, out bool exists); if (!exists) { - dict = new SelfDestructDictionary(StorageTree.EmptyBytes); + dict = new SelfDestructDictionary(StorageTree.EmptyBytes); } dict[change.StorageCell.Index] = change.Value; @@ -360,10 +362,10 @@ public void WarmUp(in StorageCell storageCell, bool isEmpty) private ReadOnlySpan LoadFromTree(in StorageCell storageCell) { - ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, storageCell.Address, out bool exists); + ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, storageCell.Address, out bool exists); if (!exists) { - dict = new SelfDestructDictionary(StorageTree.EmptyBytes); + dict = new SelfDestructDictionary(StorageTree.EmptyBytes); } ref byte[]? value = ref dict.GetValueRefOrAddDefault(storageCell.Index, out exists); @@ -455,10 +457,10 @@ public override void ClearStorage(Address address) { base.ClearStorage(address); - ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, address, out bool exists); + ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, address, out bool exists); if (!exists) { - dict = new SelfDestructDictionary(StorageTree.EmptyBytes); + dict = new SelfDestructDictionary(StorageTree.EmptyBytes); } dict.SelfDestruct(); @@ -477,10 +479,10 @@ public StorageTree Create(Address address, IScopedTrieStore trieStore, Hash256 s => new(trieStore, storageRoot, logManager); } - private class SelfDestructDictionary(TValue destructedValue) where TKey : notnull + private sealed class SelfDestructDictionary(TValue destructedValue) { private bool _selfDestruct; - private readonly Dictionary _dictionary = new(); + private readonly Dictionary _dictionary = new(Comparer.Instance); public void SelfDestruct() { @@ -488,7 +490,7 @@ public void SelfDestruct() _dictionary.Clear(); } - public ref TValue? GetValueRefOrAddDefault(TKey storageCellIndex, out bool exists) + public ref TValue? GetValueRefOrAddDefault(UInt256 storageCellIndex, out bool exists) { ref TValue value = ref CollectionsMarshal.GetValueRefOrAddDefault(_dictionary, storageCellIndex, out exists); if (!exists && _selfDestruct) @@ -499,10 +501,23 @@ public void SelfDestruct() return ref value; } - public TValue? this[TKey key] + public TValue? this[UInt256 key] { set => _dictionary[key] = value; } + + private sealed class Comparer : IEqualityComparer + { + public static Comparer Instance { get; } = new(); + + private Comparer() { } + + public bool Equals(UInt256 x, UInt256 y) + => Unsafe.As>(ref x) == Unsafe.As>(ref y); + + public int GetHashCode([DisallowNull] UInt256 obj) + => MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(in obj, 1)).FastHash(); + } } } } From 70f5995808b71d3a1a3a3c64c9bcfb906f786bf4 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 19 Sep 2024 16:12:00 +0100 Subject: [PATCH 42/42] Add memory pressure --- src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs index 63784b3f431..ca34856205e 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -44,6 +44,7 @@ static KeccakCache() // Aligned, so that no torn reads if fields of Entry are properly aligned. Memory = (Entry*)NativeMemory.AlignedAlloc(size, BitOperations.RoundUpToPowerOf2(Entry.Size)); NativeMemory.Clear(Memory, size); + GC.AddMemoryPressure((long)size); } [SkipLocalsInit]