Benchmarks Introduction (#135)

* #127 - Initial suite of benchmarks

* Finishes  two "real-world" benchmark scenarios

* Adds readme

* Simplifies logic for delayed execution

* Properly initializes LazyCache with all options not using default init - updates benchmarks

* Remove unnecessary async/await from immediately-returned Tasks

Co-authored-by: Jonas Nyrup <jnyrup@users.noreply.github.com>

* Remove unnecessary async/await from immediately-returned Tasks

Co-authored-by: Jonas Nyrup <jnyrup@users.noreply.github.com>

* Adds additional benchmarks for cache hits/misses & updates readme

Co-authored-by: Jonas Nyrup <jnyrup@users.noreply.github.com>

.gitignore

@@ -187,6 +187,16 @@ Desktop.ini
 
 # Recycle Bin used on file shares
 $RECYCLE.BIN/
-
-packages/
-.vs/
+
+packages/
+.vs/
+
+# Benchmark Dot Net
+**/BenchmarkDotNet.Artifacts/*
+**/project.lock.json
+tests/output/*
+.vs/restore.dg
+artifacts/*
+BDN.Generated
+BenchmarkDotNet.Samples/Properties/launchSettings.json
+src/BenchmarkDotNet/Disassemblers/net461/*

LazyCache.Benchmarks/BenchmarkConfig.cs

@@ -0,0 +1,22 @@
+using BenchmarkDotNet.Analysers;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
+using Perfolizer.Horology;
+
+namespace LazyCache.Benchmarks
+{
+    public class BenchmarkConfig: ManualConfig
+    {
+        public BenchmarkConfig()
+            => AddJob(Job.ShortRun)
+              .AddDiagnoser(MemoryDiagnoser.Default)
+              .AddLogger(new ConsoleLogger())
+              .AddColumn(TargetMethodColumn.Method)
+              .AddAnalyser(EnvironmentAnalyser.Default)
+              .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond));
+    }
+}

LazyCache.Benchmarks/ComplexObject.cs

@@ -0,0 +1,8 @@
+namespace LazyCache.Benchmarks
+{
+    public class ComplexObject
+    {
+        public string String { get; set; } = string.Empty;
+        public int Int { get; set; } = default;
+    }
+}

LazyCache.Benchmarks/LazyCache.Benchmarks.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <Optimize>true</Optimize>
+    <Configuration>Release</Configuration>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\LazyCache\LazyCache.csproj" />
+  </ItemGroup>
+
+</Project>

LazyCache.Benchmarks/MemoryCacheBenchmarks.cs

@@ -0,0 +1,136 @@
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using LazyCache.Providers;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace LazyCache.Benchmarks
+{
+    [Config(typeof(BenchmarkConfig))]
+    [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
+    public class MemoryCacheBenchmarks
+    { 
+        public const string CacheKey = nameof(CacheKey);
+
+        public IMemoryCache MemCache;
+        public IMemoryCache PopulatedMemCache;
+        public IAppCache AppCache;
+        public IAppCache PopulatedAppCache;
+        public ComplexObject ComplexObject;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            ComplexObject = new ComplexObject();
+
+            MemCache = new MemoryCache(new MemoryCacheOptions());
+            PopulatedMemCache = new MemoryCache(new MemoryCacheOptions());
+
+            AppCache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions())));
+            PopulatedAppCache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions())));
+
+            PopulatedAppCache.Add(CacheKey, ComplexObject);
+            PopulatedMemCache.Set(CacheKey, ComplexObject);
+        }
+
+        [GlobalCleanup]
+        public void Cleanup() => MemCache.Dispose();
+
+        /*
+         *
+         * Benchmark Cache Initialization
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory("Init")]
+        public MemoryCache DotNetMemoryCache_Init() => new MemoryCache(new MemoryCacheOptions());
+
+        [Benchmark, BenchmarkCategory("Init")]
+        public CachingService LazyCache_Init() => new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions())));
+
+        /*
+         *
+         * Benchmark Add Methods
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.Add))]
+        public void DotNetMemoryCache_Set() => MemCache.Set(CacheKey, ComplexObject);
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.Add))]
+        public void LazyCache_Set() => AppCache.Add(CacheKey, ComplexObject);
+
+        /*
+         *
+         * Benchmark Get Methods With a Cache Miss
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.Get) + "_Miss")]
+        public ComplexObject DotNetMemoryCache_Get_Miss() => MemCache.Get<ComplexObject>(CacheKey);
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.Get) + "_Miss")]
+        public ComplexObject LazyCache_Get_Miss() => AppCache.Get<ComplexObject>(CacheKey);
+
+        /*
+         *
+         * Benchmark Get Methods With a Cache Hit
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.Get) + "_Hit")]
+        public ComplexObject DotNetMemoryCache_Get_Hit() => PopulatedMemCache.Get<ComplexObject>(CacheKey);
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.Get) + "_Hit")]
+        public ComplexObject LazyCache_Get_Hit() => PopulatedAppCache.Get<ComplexObject>(CacheKey);
+
+        /*
+         *
+         * Benchmark GetOrAdd Methods With Cache Miss
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAdd) + "_Miss")]
+        public ComplexObject DotNetMemoryCache_GetOrAdd_Miss() => MemCache.GetOrCreate(CacheKey, entry => ComplexObject);
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAdd) + "_Miss")]
+        public ComplexObject LazyCache_GetOrAdd_Miss() => AppCache.GetOrAdd(CacheKey, entry => ComplexObject);
+
+        /*
+         *
+         * Benchmark GetOrAdd Methods With Cache Hit
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAdd) + "_Hit")]
+        public ComplexObject DotNetMemoryCache_GetOrAdd_Hit() => PopulatedMemCache.GetOrCreate(CacheKey, entry => ComplexObject);
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAdd) + "_Hit")]
+        public ComplexObject LazyCache_GetOrAdd_Hit() => PopulatedAppCache.GetOrAdd(CacheKey, entry => ComplexObject);
+
+        /*
+         *
+         * Benchmark GetOrAddAsync Methods With Cache Miss
+         *
+         */
+
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAddAsync) + "_Miss")]
+        public Task<ComplexObject> DotNetMemoryCache_GetOrAddAsync_Miss() => MemCache.GetOrCreateAsync(CacheKey, entry => Task.FromResult(ComplexObject));
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAddAsync) + "_Miss")]
+        public Task<ComplexObject> LazyCache_GetOrAddAsync_Miss() => AppCache.GetOrAddAsync(CacheKey, entry => Task.FromResult(ComplexObject));
+
+        /*
+         *
+         * Benchmark GetOrAddAsync Methods With Cache Hit
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAddAsync) + "_Hit")]
+        public Task<ComplexObject> DotNetMemoryCache_GetOrAddAsync_Hit() => PopulatedMemCache.GetOrCreateAsync(CacheKey, entry => Task.FromResult(ComplexObject));
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAddAsync) + "_Hit")]
+        public Task<ComplexObject> LazyCache_GetOrAddAsync_Hit() => PopulatedAppCache.GetOrAddAsync(CacheKey, entry => Task.FromResult(ComplexObject));
+    }
+}

LazyCache.Benchmarks/MemoryCacheBenchmarksRealLifeScenarios.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Management;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Reports;
+using LazyCache.Providers;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace LazyCache.Benchmarks
+{
+    [Config(typeof(BenchmarkConfig))]
+    [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
+    public class MemoryCacheBenchmarksRealLifeScenarios
+    {
+        public const string CacheKey = nameof(CacheKey);
+
+        public ComplexObject ComplexObject1;
+        public ComplexObject ComplexObject2;
+        public ComplexObject ComplexObject3;
+        public ComplexObject ComplexObject4;
+        public ComplexObject ComplexObject5;
+
+        // Trying not to introduce artificial allocations below - just measuring what the library itself needs
+        [GlobalSetup]
+        public void Setup()
+        {
+            ComplexObject1 = new ComplexObject();
+            ComplexObject2 = new ComplexObject();
+            ComplexObject3 = new ComplexObject();
+            ComplexObject4 = new ComplexObject();
+            ComplexObject5 = new ComplexObject();
+        }
+
+        [Benchmark]
+        public ComplexObject Init_CRUD()
+        {
+            var cache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions()))) as IAppCache;
+
+            cache.Add(CacheKey, ComplexObject1);
+
+            var obj = cache.Get<ComplexObject>(CacheKey);
+
+            obj.Int = 256;
+            cache.Add(CacheKey, obj);
+
+            cache.Remove(CacheKey);
+
+            return obj;
+        }
+
+        // Benchmark memory usage to ensure only a single instance of the object is created
+        // Due to the nature of AsyncLazy, this test should also only take the the time it takes to create
+        //   one instance  of the object.
+        [Benchmark]
+        public async Task<byte[]> Several_initializations_of_1Mb_object_with_200ms_delay()
+        {
+            var cache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions()))) as IAppCache;
+
+            Task AddByteArrayToCache() =>
+                cache.GetOrAddAsync(CacheKey, async () =>
+                {
+                    await Task.Delay(200);
+                    return await Task.FromResult(new byte[1024 * 1024]); // 1Mb
+                });
+
+            // Even though the second and third init attempts are later, this whole operation should still take the time of the first
+            var creationTask1 = AddByteArrayToCache(); // initialization attempt, or 200ms
+            var creationTask2 = Delayed(50, AddByteArrayToCache);
+            var creationTask3 = Delayed(50, AddByteArrayToCache);
+
+            await Task.WhenAll(creationTask1, creationTask2, creationTask3);
+
+            return cache.Get<byte[]>(CacheKey);
+        }
+
+        private async Task Delayed(int ms, Func<Task> action)
+        {
+            await Task.Delay(ms);
+            await action();
+        }
+    }
+}

LazyCache.Benchmarks/Program.cs

@@ -0,0 +1,9 @@
+using BenchmarkDotNet.Running;
+
+namespace LazyCache.Benchmarks
+{
+    public static class Program
+    {
+        public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+    }
+}

LazyCache.Benchmarks/Properties/launchSettings.json

@@ -0,0 +1,7 @@
+{
+  "profiles": {
+    "LazyCache.Benchmarks": {
+      "commandName": "Project"
+    }
+  }
+}

LazyCache.Benchmarks/README.md

@@ -0,0 +1,73 @@
+# LazyCache.Benchmarks
+This project is dedicated towards benchmarking (using [BenchmarkDotNet](https://benchmarkdotnet.org/index.html)) the basic functionality of LazyCache such that contributors and maintainers can verify the efficacy of changes towards the project - for better or for worse.
+
+## Note to readers
+While it is always a good idea to understand performance of your third party libraries, it is rare that you will be concerned with performance on the scale of nanoseconds such that this library operates on. Be wary of premature optimization.
+
+# How to run
+- Ensure you have the requisite dotnet SDKs found in _LazyCache.Benchmarks.csproj_
+- Clone the project
+- Open your favorite terminal, navigate to the Benchmark Project
+- `dotnet run -c Release`
+- Pick your desired benchmark suite via numeric entry
+
+If you are interested in benchmarking a specific method (after making changes to it, for instance), you can conveniently filter down to one specific benchmark, e.g. `dotnet run -c Release -- -f *Get` will only run the benchmarks for `IAppCache.Get` implementations, likewise with `*GetOrAddAsync`, or other methods.
+
+# Contributing
+If you have ideas for one or more benchmarks not covered here, please add an issue describing what you would like to see. Pull requests are always welcome!
+
+# Benchmark Types
+There are two types of benchmarks available.
+
+## Basics
+The basic benchmarks are small and laser-focused on testing individual aspects of LazyCache. This suite of benchmarks uses the out-of-the-box MemoryCache from dotnet [seen here](https://github.com/dotnet/runtime/blob/master/src/libraries/Microsoft.Extensions.Caching.Memory/src/) as a baseline, to demonstrate the "cost" of LazyCache in comparison.
+
+## Integration
+These benchmarks are designed to showcase full use-cases of LazyCache by chaining together various operations. As an example, with the Memory Diagnoser from BenchmarkDotNet, we can verify that concurrent calls to initialize a cache item correctly spin up one instance of said item, with the subsequent calls awaiting its result.
+
+### Gotchas
+Remember that BenchmarkDotNet dutifully monitors allocations inside the benchmark method, and _only_ the method. At the time of writing, the default instance of the MemoryCacheProvider is static, and allocations into this cache will **not** be monitored by BenchmarkDotNet. For all benchmarks, please ensure you are creating new instances of the Service, Provider, and backing Cache.
+
+# Benchmarks
+
+```
+// * Summary *
+BenchmarkDotNet=v0.12.1, OS=Windows 10.0.18362.1082 (1903/May2019Update/19H1)
+AMD Ryzen 9 3900X, 1 CPU, 24 logical and 12 physical cores
+.NET Core SDK=5.0.100-preview.7.20366.6
+  [Host]   : .NET Core 3.1.7 (CoreCLR 4.700.20.36602, CoreFX 4.700.20.37001), X64 RyuJIT
+  ShortRun : .NET Core 3.1.7 (CoreCLR 4.700.20.36602, CoreFX 4.700.20.37001), X64 RyuJIT
+
+Job=ShortRun  IterationCount=3  LaunchCount=1
+WarmupCount=3
+```
+|                               Method |       Mean |       Error |   StdDev | Ratio | RatioSD |  Gen 0 |  Gen 1 |  Gen 2 | Allocated |
+|------------------------------------- |-----------:|------------:|---------:|------:|--------:|-------:|-------:|-------:|----------:|
+|               DotNetMemoryCache_Init | 1,814.2 ns | 1,080.95 ns | 59.25 ns |  1.00 |    0.00 | 0.1850 | 0.0916 | 0.0019 |    1560 B |
+|                       LazyCache_Init | 3,265.5 ns |   599.75 ns | 32.87 ns |  1.80 |    0.07 | 0.3090 | 0.1526 |      - |    2600 B |
+|                                      |            |             |          |       |         |        |        |        |           |
+|                DotNetMemoryCache_Set |   504.1 ns |    42.38 ns |  2.32 ns |  1.00 |    0.00 | 0.0496 |      - |      - |     416 B |
+|                        LazyCache_Set |   841.6 ns |   172.51 ns |  9.46 ns |  1.67 |    0.02 | 0.0801 |      - |      - |     672 B |
+|                                      |            |             |          |       |         |        |        |        |           |
+|           DotNetMemoryCache_Get_Miss |   201.1 ns |     3.54 ns |  0.19 ns |  1.00 |    0.00 |      - |      - |      - |         - |
+|                   LazyCache_Get_Miss |   241.1 ns |    13.94 ns |  0.76 ns |  1.20 |    0.00 |      - |      - |      - |         - |
+|                                      |            |             |          |       |         |        |        |        |           |
+|            DotNetMemoryCache_Get_Hit |   242.2 ns |    28.93 ns |  1.59 ns |  1.00 |    0.00 |      - |      - |      - |         - |
+|                    LazyCache_Get_Hit |   280.4 ns |    10.45 ns |  0.57 ns |  1.16 |    0.01 |      - |      - |      - |         - |
+|                                      |            |             |          |       |         |        |        |        |           |
+|      DotNetMemoryCache_GetOrAdd_Miss |   269.9 ns |     6.57 ns |  0.36 ns |  1.00 |    0.00 | 0.0076 |      - |      - |      64 B |
+|              LazyCache_GetOrAdd_Miss |   368.5 ns |    60.35 ns |  3.31 ns |  1.37 |    0.01 | 0.0191 |      - |      - |     160 B |
+|                                      |            |             |          |       |         |        |        |        |           |
+|       DotNetMemoryCache_GetOrAdd_Hit |   269.1 ns |     4.48 ns |  0.25 ns |  1.00 |    0.00 | 0.0076 |      - |      - |      64 B |
+|               LazyCache_GetOrAdd_Hit |   377.1 ns |    10.57 ns |  0.58 ns |  1.40 |    0.00 | 0.0191 |      - |      - |     160 B |
+|                                      |            |             |          |       |         |        |        |        |           |
+| DotNetMemoryCache_GetOrAddAsync_Miss |   312.7 ns |    53.05 ns |  2.91 ns |  1.00 |    0.00 | 0.0162 |      - |      - |     136 B |
+|         LazyCache_GetOrAddAsync_Miss |   507.5 ns |    33.96 ns |  1.86 ns |  1.62 |    0.02 | 0.0362 |      - |      - |     304 B |
+|                                      |            |             |          |       |         |        |        |        |           |
+|  DotNetMemoryCache_GetOrAddAsync_Hit |   314.5 ns |    65.34 ns |  3.58 ns |  1.00 |    0.00 | 0.0162 |      - |      - |     136 B |
+|          LazyCache_GetOrAddAsync_Hit |   535.9 ns |    47.83 ns |  2.62 ns |  1.70 |    0.03 | 0.0448 |      - |      - |     376 B |
+
+|                                                 Method |             Mean |           Error |          StdDev |  Gen 0 |  Gen 1 |  Gen 2 |  Allocated |
+|------------------------------------------------------- |-----------------:|----------------:|----------------:|-------:|-------:|-------:|-----------:|
+|                                              Init_CRUD |       5,115.1 ns |        991.0 ns |        54.32 ns | 0.4730 | 0.2365 | 0.0076 |     3.9 KB |
+| Several_initializations_of_1Mb_object_with_200ms_delay | 207,329,988.9 ns | 31,342,899.9 ns | 1,718,010.11 ns |      - |      - |      - | 1031.75 KB |