Fix BTree deadlock on split. (#36)

src/Playground/DeadlockFinder.cs

@@ -0,0 +1,111 @@
+using System.Diagnostics;
+using Tenray.ZoneTree.Collections.BTree;
+using Tenray.ZoneTree.Collections.BTree.Lock;
+using Tenray.ZoneTree.Comparers;
+
+public class DeadlockFinder
+{
+    BTree<long, long> Tree;
+
+    Stopwatch Stopwatch;
+
+    public bool UpsertOnlyMode = true;
+
+    void TreeLoop(int taskNo)
+    {
+        var tree = Tree;
+        try
+        {
+            const int recCount = 100_000_000;
+            const int upsertLogFrequency = 1_000_000;
+            const int iterationLogFrequency = 10_000_000;
+            const int iterationYieldFrequency = 1000;
+            var rand = new Random();
+
+            void resetTree()
+            {
+                if (Stopwatch.Elapsed.TotalSeconds > 5 && tree.Length > recCount * 0.1)
+                {
+                    {
+                        Console.WriteLine("Resetting tree." + tree.Length);
+                        Stopwatch.Restart();
+                        tree = new BTree<long, long>(new Int64ComparerAscending(), BTreeLockMode.NodeLevelMonitor);
+                    }
+                }
+            }
+            if (UpsertOnlyMode || taskNo % 3 == 0)
+            {
+                uint i = 0;
+                while (true)
+                {
+                    if (taskNo == 0)
+                        resetTree();
+                    tree.Upsert(rand.Next() % recCount, 3, out _);
+                    ++i;
+                    if (i % upsertLogFrequency == 0)
+                    {
+                        Console.WriteLine($"Upsert():{i}      task :{taskNo}");
+                    }
+                }
+            }
+            else if (taskNo % 3 == 1)
+            {
+                uint i = 0;
+                while (true)
+                {
+                    var iterator = tree.GetFirstIterator();
+                    while (iterator.HasNext())
+                    {
+                        iterator.Next();
+                        ++i;
+                        if (rand.Next() % iterationYieldFrequency == 1)
+                            Thread.Yield();
+                        if (i % iterationLogFrequency == 0)
+                        {
+                            Console.WriteLine($"Next():{i}      task :{taskNo}");
+                        }
+                    }
+                }
+            }
+            else
+            {
+                uint i = 0;
+                while (true)
+                {
+                    var iterator = tree.GetLastIterator();
+                    while (iterator.HasPrevious())
+                    {
+                        iterator.Previous();
+                        ++i;
+                        if (rand.Next() % iterationYieldFrequency == 1)
+                            Thread.Yield();
+
+                        if (i % iterationLogFrequency == 0)
+                        {
+                            Console.WriteLine($"Previous():{i}      task :{taskNo}");
+                        }
+                    }
+                }
+            }
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine(e.ToString());
+        }
+    }
+
+    public void RunDeadLockFinder()
+    {
+        Tree = new BTree<long, long>(new Int64ComparerAscending(), BTreeLockMode.NodeLevelMonitor);
+        Stopwatch = Stopwatch.StartNew();
+        Thread t0 = null;
+        for (var i = 0; i < 100; ++i)
+        {
+            var k = i;
+            var t = new Thread(() => TreeLoop(k));
+            t.Start();
+            t0 = t;
+        }
+        t0.Join();
+    }
+}

src/Playground/Program.cs

@@ -1,11 +1,6 @@
-using BenchmarkDotNet.Running;
-using Playground;
-using Playground.Benchmark;
-using Playground.InMemoryTreeBenchmark;
-using Tenray.ZoneTree.Core;
+using Playground.Benchmark;
 using Tenray.ZoneTree.Logger;
 using Tenray.ZoneTree.Options;
-using Tenray.ZoneTree.WAL;
 
 TestConfig.EnableIncrementalBackup = false;
 TestConfig.MutableSegmentMaxItemCount = 1_000_000;
@@ -113,4 +108,10 @@
     b.Run(steve.Iterate);
     b.Run(steve.ShowBottomSegments);
     b.Run(steve.MergeBottomSegments);
+}
+
+if (testCase == 5)
+{
+    // Enable LOG_DEADLOCK_PREVENTION preprocessor on BTree.Write to see deadlock logs.
+    new DeadlockFinder().RunDeadLockFinder();
 }

src/ZoneTree/Collections/BTree/BTree.Node.cs

@@ -21,7 +21,7 @@ public class Node
 
 #if USE_NODE_IDS
         static int IncrementalId = 0;
-      
+
         public int Id;
 #endif
 

src/ZoneTree/Collections/BTree/BTree.Write.cs

@@ -1,4 +1,6 @@
-using Tenray.ZoneTree.Exceptions;
+#undef LOG_DEADLOCK_PREVENTION
+
+using Tenray.ZoneTree.Exceptions;
 
 namespace Tenray.ZoneTree.Collections.BTree;
 
@@ -34,7 +36,7 @@ public bool Upsert(in TKey key, in TValue value, out long opIndex)
                 var newRoot = new Node(GetNodeLocker(), NodeSize);
                 newRoot.Children[0] = root;
                 newRoot.WriteLock();
-                SplitChild(newRoot, 0, root);
+                TrySplitChild(newRoot, 0, root);
                 var result = UpsertNonFull(newRoot, in key, in value, out opIndex);
                 Root = newRoot;
                 root.WriteUnlock();
@@ -75,14 +77,14 @@ public bool TryInsert(in TKey key, in TValue value, out long opIndex)
                 var newRoot = new Node(GetNodeLocker(), NodeSize);
                 newRoot.Children[0] = root;
                 newRoot.WriteLock();
-                SplitChild(newRoot, 0, root);
+                TrySplitChild(newRoot, 0, root);
                 var result = TryInsertNonFull(newRoot, in key, in value, out opIndex);
                 Root = newRoot;
                 root.WriteUnlock();
                 return result;
             }
         }
-        catch(Exception)
+        catch (Exception)
         {
             Root.WriteUnlock();
             throw;
@@ -124,7 +126,7 @@ public AddOrUpdateResult AddOrUpdate(
                 var newRoot = new Node(GetNodeLocker(), NodeSize);
                 newRoot.Children[0] = root;
                 newRoot.WriteLock();
-                SplitChild(newRoot, 0, root);
+                TrySplitChild(newRoot, 0, root);
                 AddOrUpdateResult result =
                     TryAddOrUpdateNonFull(newRoot, in key, adder, updater, out opIndex);
                 Root = newRoot;
@@ -143,7 +145,7 @@ public AddOrUpdateResult AddOrUpdate(
         }
     }
 
-    void SplitChild(Node parent, int parentInsertPosition, Node child)
+    bool TrySplitChild(Node parent, int parentInsertPosition, Node child)
     {
         var pivotPosition = (child.Length + 1) / 2;
         ref var pivotKey = ref child.Keys[pivotPosition];
@@ -170,56 +172,43 @@ void SplitChild(Node parent, int parentInsertPosition, Node child)
             // SOLUTION:
             // TRY TO LOCK NEXT WITH TIMEOUT
             // IF CAN NOT LOCK IN TIME UNLOCK PRE
-            // THEN SLEEP SOME
-            // THEN RELOCK PRE
-            // AND TRY TO LOCK NEXT ONCE AGAIN IN A LOOP.
+            // THEN RETURN FALSE AND CALLER WILL TRY AGAIN
             var lockTimeout = 500;
             var next = childLeaf.Next;
             var isNextLocked = true;
-            while(true)
+
+            if (next != null)
+                isNextLocked = next.TryEnterWriteLock(lockTimeout);
+            if (!isNextLocked)
             {
-                if (next != null)
-                    isNextLocked = next.TryEnterWriteLock(lockTimeout);
-                if (isNextLocked)
-                    break;
+#if LOG_DEADLOCK_PREVENTION
+                Console.WriteLine("Avoiding deadlock scenario 1!");
+#endif
                 pre?.WriteUnlock();
-                Thread.Sleep(100);
-                pre?.WriteLock();
-                while (childLeaf.Previous != pre)
-                {
-                    pre.WriteUnlock();
-                    pre = childLeaf.Previous;
-                    pre.WriteLock();
-                }
+                return false;
             }
 
             // prevent neighbour split at the same time.
             while (childLeaf.Next != next)
             {
-                next.WriteUnlock();
+                next?.WriteUnlock();
                 next = childLeaf.Next;
                 isNextLocked = true;
-                while (true)
+                if (next != null)
+                    isNextLocked = next.TryEnterWriteLock(lockTimeout);
+                if (!isNextLocked)
                 {
-                    if (next != null)
-                        isNextLocked = next.TryEnterWriteLock(lockTimeout);
-                    if (isNextLocked)
-                        break;
+#if LOG_DEADLOCK_PREVENTION
+                    Console.WriteLine("Avoiding deadlock scenario 2!");
+#endif
                     pre?.WriteUnlock();
-                    Thread.Sleep(100);
-                    pre?.WriteLock();
-                    while (childLeaf.Previous != pre)
-                    {
-                        pre.WriteUnlock();
-                        pre = childLeaf.Previous;
-                        pre.WriteLock();
-                    }
+                    return false;
                 }
             }
 
             left.Previous = pre;
             right.Next = next;
-            
+
             if (pre == null)
                 FirstLeafNode = left;
             else
@@ -241,6 +230,7 @@ void SplitChild(Node parent, int parentInsertPosition, Node child)
                 .Split(pivotPosition, NodeSize, GetNodeLocker(), GetNodeLocker());
             parent.InsertKeyAndChild(parentInsertPosition, in pivotKey, left, right);
         }
+        return true;
     }
 
     bool UpsertNonFull(Node node, in TKey key, in TValue value, out long opIndex)
@@ -268,8 +258,12 @@ bool UpsertNonFull(Node node, in TKey key, in TValue value, out long opIndex)
             child.WriteLock();
             if (child.IsFull)
             {
-                SplitChild(node, position, child);
+                var splitted = TrySplitChild(node, position, child);
                 child.WriteUnlock();
+                if (!splitted)
+                {
+                    continue;
+                }
 
                 if (Comparer.Compare(in key, in node.Keys[position]) >= 0)
                     ++position;
@@ -310,8 +304,12 @@ bool TryInsertNonFull(Node node, in TKey key, in TValue value, out long opIndex)
             child.WriteLock();
             if (child.IsFull)
             {
-                SplitChild(node, position, child);
+                var splitted = TrySplitChild(node, position, child);
                 child.WriteUnlock();
+                if (!splitted)
+                {
+                    continue;
+                }
 
                 if (Comparer.Compare(in key, in node.Keys[position]) >= 0)
                     ++position;
@@ -325,7 +323,7 @@ bool TryInsertNonFull(Node node, in TKey key, in TValue value, out long opIndex)
     }
 
     AddOrUpdateResult TryAddOrUpdateNonFull(
-        Node node, in TKey key, AddDelegate adder, UpdateDelegate updater, 
+        Node node, in TKey key, AddDelegate adder, UpdateDelegate updater,
         out long opIndex)
     {
         while (true)
@@ -355,8 +353,12 @@ AddOrUpdateResult TryAddOrUpdateNonFull(
             child.WriteLock();
             if (child.IsFull)
             {
-                SplitChild(node, position, child);
+                var splitted = TrySplitChild(node, position, child);
                 child.WriteUnlock();
+                if (!splitted)
+                {
+                    continue;
+                }
 
                 if (Comparer.Compare(in key, in node.Keys[position]) >= 0)
                     ++position;

src/ZoneTree/ZoneTree.csproj

@@ -47,7 +47,7 @@
     </PackageReference>
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="K4os.Compression.LZ4" Version="1.2.16" />
+    <PackageReference Include="K4os.Compression.LZ4" Version="1.3.5" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>