Exceptions: Add threadpool stats in .NET Core+ (#1964)

I swear I already did this somewhere but it got lost, this adds additional thread pool stats exposed in .NET Core 3.1+. We'll get for example ThreadPool.PendingWorkItemCount on exception messages to better indicate app contention.

docs/ReleaseNotes.md

@@ -8,6 +8,7 @@
 - Fix profiler showing `EVAL` instead `EVALSHA` (#1930 via martinpotter)
 - Moved tiebreaker fetching in connections into the handshake phase (streamline + simplification) (#1931 via NickCraver)
 - Fixed potential disposed object usage around Arenas (pulling in [Piplines.Sockets.Unofficial#63](https://github.com/mgravell/Pipelines.Sockets.Unofficial/pull/63) by MarcGravell)
+- Adds thread pool work item stats to exception messages to help diagnose contention (#1964 via NickCraver)
 
 ## 2.2.88
 

docs/Timeouts.md

@@ -94,6 +94,7 @@ By default Redis Timeout exception(s) includes useful information, which can hel
 |mgr | 8 of 10 available|Redis Internal Dedicated Thread Pool State| 
 |IOCP | IOCP: (Busy=0,Free=500,Min=248,Max=500)| Runtime Global Thread Pool IO Threads. |
 |WORKER | WORKER: (Busy=170,Free=330,Min=248,Max=500)| Runtime Global Thread Pool Worker Threads.| 
+|POOL | POOL: (Threads=8,QueuedItems=0,CompletedItems=42)| Thread Pool Work Item Stats.| 
 |v | Redis Version: version |Current redis version you are currently using in your application.|
 |active | Message-Current: {string} |Included in exception message when `IncludeDetailInExceptions=True` on multiplexer|
 |next | Message-Next: {string} |When `IncludeDetailInExceptions=True` on multiplexer, it might include command and key, otherwise only command.|

src/StackExchange.Redis/ConnectionMultiplexer.cs

@@ -702,8 +702,12 @@ private static void LogWithThreadPoolStats(LogProxy log, string message, out int
             {
                 var sb = new StringBuilder();
                 sb.Append(message);
-                busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
+                busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker, out string workItems);
                 sb.Append(", IOCP: ").Append(iocp).Append(", WORKER: ").Append(worker);
+                if (workItems != null)
+                {
+                    sb.Append(", POOL: ").Append(workItems);
+                }
                 log?.WriteLine(sb.ToString());
             }
         }

src/StackExchange.Redis/ExceptionFactory.cs

@@ -353,9 +353,13 @@ ServerEndPoint server
                     Add(data, sb, "Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(multiplexer.ServerSelectionStrategy).ToString());
                 }
             }
-            int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
+            int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker, out string workItems);
             Add(data, sb, "ThreadPool-IO-Completion", "IOCP", iocp);
             Add(data, sb, "ThreadPool-Workers", "WORKER", worker);
+            if (workItems != null)
+            {
+                Add(data, sb, "ThreadPool-Items", "POOL", workItems);
+            }
             data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
 
             if (multiplexer.IncludePerformanceCountersInExceptions)

src/StackExchange.Redis/PerfCounterHelper.cs

@@ -56,17 +56,17 @@ public static bool TryGetSystemCPU(out float value)
 
         internal static string GetThreadPoolAndCPUSummary(bool includePerformanceCounters)
         {
-            GetThreadPoolStats(out string iocp, out string worker);
+            GetThreadPoolStats(out string iocp, out string worker, out string workItems);
             var cpu = includePerformanceCounters ? GetSystemCpuPercent() : "n/a";
-            return $"IOCP: {iocp}, WORKER: {worker}, Local-CPU: {cpu}";
+            return $"IOCP: {iocp}, WORKER: {worker}, POOL: {workItems ?? "n/a"}, Local-CPU: {cpu}";
         }
 
         internal static string GetSystemCpuPercent() =>
             RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && TryGetSystemCPU(out float systemCPU)
                 ? Math.Round(systemCPU, 2) + "%"
                 : "unavailable";
 
-        internal static int GetThreadPoolStats(out string iocp, out string worker)
+        internal static int GetThreadPoolStats(out string iocp, out string worker, out string workItems)
         {
             ThreadPool.GetMaxThreads(out int maxWorkerThreads, out int maxIoThreads);
             ThreadPool.GetAvailableThreads(out int freeWorkerThreads, out int freeIoThreads);
@@ -77,6 +77,13 @@ internal static int GetThreadPoolStats(out string iocp, out string worker)
 
             iocp = $"(Busy={busyIoThreads},Free={freeIoThreads},Min={minIoThreads},Max={maxIoThreads})";
             worker = $"(Busy={busyWorkerThreads},Free={freeWorkerThreads},Min={minWorkerThreads},Max={maxWorkerThreads})";
+
+#if NETCOREAPP
+            workItems = $"(Threads={ThreadPool.ThreadCount},QueuedItems={ThreadPool.PendingWorkItemCount},CompletedItems={ThreadPool.CompletedWorkItemCount})";
+#else
+            workItems = null;
+#endif
+
             return busyWorkerThreads;
         }
     }

tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs

@@ -123,6 +123,11 @@ public void TimeoutException()
                     Assert.Contains("inst: 0, qu: 0, qs: 0, aw: False, in: 0, in-pipe: 0, out-pipe: 0", ex.Message);
                     Assert.Contains("mc: 1/1/0", ex.Message);
                     Assert.Contains("serverEndpoint: " + server.EndPoint, ex.Message);
+                    Assert.Contains("IOCP: ", ex.Message);
+                    Assert.Contains("WORKER: ", ex.Message);
+#if NETCOREAPP
+                    Assert.Contains("POOL: ", ex.Message);
+#endif
                     Assert.DoesNotContain("Unspecified/", ex.Message);
                     Assert.EndsWith(" (Please take a look at this article for some common client-side issues that can cause timeouts: https://stackexchange.github.io/StackExchange.Redis/Timeouts)", ex.Message);
                     Assert.Null(ex.InnerException);