From 8095973d61f46295c0f0aef5a1e901787808c292 Mon Sep 17 00:00:00 2001
From: Ben Adams <thundercat@illyriad.co.uk>
Date: Fri, 16 Aug 2024 15:32:09 +0100
Subject: [PATCH] Reduce contention in ClockCache

---
 src/Nethermind/Nethermind.Core/Caching/ClockCache.cs | 12 +++++++++---
 .../Nethermind.Core/Caching/ClockCacheBase.cs        |  3 +++
 .../Nethermind.Core/Caching/ClockKeyCache.cs         | 12 +++++++++---
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/Nethermind/Nethermind.Core/Caching/ClockCache.cs b/src/Nethermind/Nethermind.Core/Caching/ClockCache.cs
index caa528b6091..2065e3cad9f 100644
--- a/src/Nethermind/Nethermind.Core/Caching/ClockCache.cs
+++ b/src/Nethermind/Nethermind.Core/Caching/ClockCache.cs
@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 
 using Nethermind.Core.Threading;
@@ -74,7 +75,8 @@ private bool SetSlow(TKey key, TValue val)
             return false;
         }
 
-        int offset = _cacheMap.Count;
+        int offset = _count;
+        Debug.Assert(_cacheMap.Count == _count);
         if (FreeOffsets.Count > 0)
         {
             offset = FreeOffsets.Dequeue();
@@ -86,6 +88,8 @@ private bool SetSlow(TKey key, TValue val)
 
         _cacheMap[key] = new LruCacheItem(offset, val);
         KeyToOffset[offset] = key;
+        _count++;
+        Debug.Assert(_cacheMap.Count == _count);
 
         return true;
     }
@@ -93,7 +97,7 @@ private bool SetSlow(TKey key, TValue val)
     private int Replace(TKey key)
     {
         int position = Clock;
-        int max = _cacheMap.Count;
+        int max = _count;
         while (true)
         {
             if (position >= max)
@@ -108,6 +112,7 @@ private int Replace(TKey key)
                 {
                     ThrowInvalidOperationException();
                 }
+                _count--;
                 break;
             }
 
@@ -132,6 +137,7 @@ public bool Delete(TKey key)
 
         if (_cacheMap.Remove(key, out LruCacheItem? ov))
         {
+            _count--;
             KeyToOffset[ov.Offset] = default;
             ClearAccessed(ov.Offset);
             FreeOffsets.Enqueue(ov.Offset);
@@ -157,7 +163,7 @@ public bool Contains(TKey key)
         return _cacheMap.ContainsKey(key);
     }
 
-    public int Count => _cacheMap.Count;
+    public int Count => _count;
 
     private class LruCacheItem(int offset, TValue v)
     {
diff --git a/src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs b/src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs
index bea020a9505..f2884acdc95 100644
--- a/src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs
+++ b/src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs
@@ -21,6 +21,8 @@ public abstract class ClockCacheBase<TKey>
     protected Queue<int> FreeOffsets { get; } = new();
 
     protected int Clock { get; set; } = 0;
+    // Use local count to avoid lock contention with reads on ConcurrentDictionary.Count
+    protected int _count = 0;
 
     protected ClockCacheBase(int maxCapacity)
     {
@@ -35,6 +37,7 @@ protected void Clear()
     {
         if (MaxCapacity == 0) return;
 
+        _count = 0;
         Clock = 0;
         FreeOffsets.Clear();
         KeyToOffset.AsSpan().Clear();
diff --git a/src/Nethermind/Nethermind.Core/Caching/ClockKeyCache.cs b/src/Nethermind/Nethermind.Core/Caching/ClockKeyCache.cs
index 9979255f204..0b510231834 100644
--- a/src/Nethermind/Nethermind.Core/Caching/ClockKeyCache.cs
+++ b/src/Nethermind/Nethermind.Core/Caching/ClockKeyCache.cs
@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 
 using Nethermind.Core.Threading;
@@ -51,7 +52,8 @@ private bool SetSlow(TKey key)
             return false;
         }
 
-        offset = _cacheMap.Count;
+        offset = _count;
+        Debug.Assert(_cacheMap.Count == _count);
         if (FreeOffsets.Count > 0)
         {
             offset = FreeOffsets.Dequeue();
@@ -63,6 +65,8 @@ private bool SetSlow(TKey key)
 
         _cacheMap[key] = offset;
         KeyToOffset[offset] = key;
+        _count++;
+        Debug.Assert(_cacheMap.Count == _count);
 
         return true;
     }
@@ -70,7 +74,7 @@ private bool SetSlow(TKey key)
     private int Replace(TKey key)
     {
         int position = Clock;
-        int max = _cacheMap.Count;
+        int max = _count;
         while (true)
         {
             if (position >= max)
@@ -85,6 +89,7 @@ private int Replace(TKey key)
                 {
                     ThrowInvalidOperationException();
                 }
+                _count--;
                 break;
             }
 
@@ -108,6 +113,7 @@ public bool Delete(TKey key)
 
         if (_cacheMap.Remove(key, out int offset))
         {
+            _count--;
             ClearAccessed(offset);
             FreeOffsets.Enqueue(offset);
             return true;
@@ -131,5 +137,5 @@ public bool Contains(TKey key)
         return _cacheMap.ContainsKey(key);
     }
 
-    public int Count => _cacheMap.Count;
+    public int Count => _count;
 }