[Profiler] Add gen2 leak scenario (#6071)

## Summary of changes
Change the LOH fragmentation scenario into a gen2 leak

## Reason for change
Simpler to understand and avoid too fast memory increase that leads to
OOM/app crash

## Implementation details
Remove the large integer array from the controller and create dedicated
News request threads

## Test coverage
N/A

## Other details
<!-- Fixes #{issue} -->

<!-- ⚠️ Note: where possible, please obtain 2 approvals prior to
merging. Unless CODEOWNERS specifies otherwise, for external teams it is
typically best to have one review from a team member, and one review
from apm-dotnet. Trivial changes do not require 2 reviews. -->

profiler/src/Demos/Samples.BuggyBits/Controllers/NewsController.cs

@@ -5,35 +5,51 @@
 
 using System;
 using System.Collections.Generic;
+using System.Threading;
 using BuggyBits.Models;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Caching.Memory;
 
 namespace BuggyBits.Controllers
 {
+
     public class NewsController : Controller
     {
-#pragma warning disable IDE0052 // Remove unread private members | this field is used to better show memory leaks
-        private readonly int[] bits = new int[25000];
-#pragma warning restore IDE0052
+        private static int _id = 0;
+        private int _instanceId;
+
+//#pragma warning disable IDE0052 // Remove unread private members | this field is used to better show memory leaks
+//        private readonly int[] bits = new int[25000];
+//#pragma warning restore IDE0052
         private IMemoryCache cache;
+        private DateTime _creationTime;
 
         public NewsController(IMemoryCache cache)
         {
+            _creationTime = DateTime.Now;
+            _instanceId = Interlocked.Increment(ref _id);
             this.cache = cache;
             GC.Collect();
         }
 
+        ~NewsController()
+        {
+            Console.WriteLine($"{DateTime.Now.ToShortTimeString()} | {(DateTime.Now -_creationTime).TotalSeconds, 4} - ~NewsController #{_instanceId}");
+        }
+
         public IActionResult Index()
         {
             string key = Guid.NewGuid().ToString();
             var cachedResult = cache.GetOrCreate(key, cacheEntry =>
             {
-                cacheEntry.SlidingExpiration = TimeSpan.FromMinutes(5);
+                ////Adding a sliding expiration will help to evict cache entries sooner
+                ////but the LOH will become fragmented
+                //cacheEntry.SlidingExpiration = TimeSpan.FromMinutes(5);
+
                 cacheEntry.RegisterPostEvictionCallback(CacheRemovedCallback);
                 cacheEntry.Priority = CacheItemPriority.NeverRemove;
 
-                return new string("New site launched 2008-02-02");
+                return new string($"New site launched " + DateTime.Now);
             });
 
             var news = new List<News>
@@ -45,6 +61,10 @@ public IActionResult Index()
 
         private void CacheRemovedCallback(object key, object value, EvictionReason reason, object state)
         {
+            if (reason == EvictionReason.Capacity)
+            {
+                Console.WriteLine($"Cache entry {key} = '{value}' was removed due to capacity");
+            }
         }
     }
 }

profiler/src/Demos/Samples.BuggyBits/SelfInvoker.cs

@@ -20,13 +20,14 @@ public class SelfInvoker : IDisposable
         private readonly HttpClient _httpClient;
         private readonly Scenario _scenario;
         private readonly int _nbIdleThreads;
-
-        public SelfInvoker(CancellationToken token, Scenario scenario, int nbIdleThreds)
+        private readonly int _nbNewsThreads;
+        public SelfInvoker(CancellationToken token, Scenario scenario, int nbIdleThreads)
         {
             _exitToken = token;
             _httpClient = new HttpClient();
             _scenario = scenario;
-            _nbIdleThreads = nbIdleThreds;
+            _nbIdleThreads = nbIdleThreads;
+            _nbNewsThreads = 6;
         }
 
         public void Dispose()
@@ -46,6 +47,8 @@ public async Task RunAsync(string rootUrl, int iterations = 0)
             }
             else
             {
+                StartThreadsForLeaks(rootUrl, iterations);
+
                 try
                 {
                     List<string> asyncEndpoints = GetEndpoints(rootUrl);
@@ -75,6 +78,49 @@ public async Task RunAsync(string rootUrl, int iterations = 0)
             Console.WriteLine($"{this.GetType().Name} stopped.");
         }
 
+        private void StartThreadsForLeaks(string rootUrl, int iterations)
+        {
+            if ((_scenario & Scenario.MemoryLeak) == 0)
+            {
+                return;
+            }
+
+            // Each LOH region is 32 MB so if we want to see the RSS/committed bytes to grow,
+            // it is needed to allocate 32 x 100 KB. But this is not correct because the GC will
+            // use 1 LOH heap per core. So, we need to allocate 32 x 100 KB x nbCores.
+            // Let's tune it for a slower growth. Remove the LOH buffer to allow longer lived application.
+            // Note: use DOTNET_GCHeapCount=nbHeaps to set a limited number of heaps and see the RSS/Private bytes grow.
+            // Also, we don't want to allocate the same object too many time in a row to avoid
+            // forcing the sampling of these allocations.
+            for (var i = 0; i < _nbNewsThreads; i++)
+            {
+                Task.Factory.StartNew(
+                    async () =>
+                    {
+                        var guid = Guid.NewGuid();
+                        var bytes = guid.ToByteArray();
+                        int randomIncrement = bytes[0];
+                        TimeSpan delay = TimeSpan.FromMilliseconds(300 + randomIncrement);
+                        Console.WriteLine($"Delay for leak = {delay}");
+                        await Task.Delay(delay);
+
+                        // Run for the given number of iterations
+                        // 0 means wait for cancellation
+                        int current = 0;
+                        while (
+                            ((iterations == 0) && !_exitToken.IsCancellationRequested) ||
+                            (iterations > current))
+                        {
+                            await ExecuteIterationAsync($"{rootUrl}/News");
+                            await Task.Delay(TimeSpan.FromMilliseconds(1000 + delay.TotalMilliseconds));
+                            current++;
+                        }
+
+                    },
+                    creationOptions: TaskCreationOptions.LongRunning);
+            }
+        }
+
         private void CreateIdleThreads()
         {
             if (_nbIdleThreads == 0)
@@ -129,11 +175,6 @@ private List<string> GetEndpoints(string rootUrl)
                     urls.Add($"{rootUrl}/Products/ParallelLock");
                 }
 
-                if ((_scenario & Scenario.MemoryLeak) == Scenario.MemoryLeak)
-                {
-                    urls.Add($"{rootUrl}/News");
-                }
-
                 if ((_scenario & Scenario.EndpointsCount) == Scenario.EndpointsCount)
                 {
                     urls.Add($"{rootUrl}/End.Point.With.Dots");
@@ -163,7 +204,12 @@ private async Task ExecuteIterationAsync(string endpointUrl)
                 string responsePayload = await response.Content.ReadAsStringAsync();
                 int responseLen = responsePayload.Length;
                 sw.Stop();
-                Console.WriteLine($"{endpointUrl} | response length = {responseLen} in {sw.ElapsedMilliseconds} ms");
+
+                // hide traces when memory leaks
+                if ((_scenario & Scenario.MemoryLeak) == 0)
+                {
+                    Console.WriteLine($"{endpointUrl} | response length = {responseLen} in {sw.ElapsedMilliseconds} ms");
+                }
             }
             catch (TaskCanceledException)
             {