Benchmarks Introduction

.gitignore

@@ -187,6 +187,16 @@ Desktop.ini
 
 # Recycle Bin used on file shares
 $RECYCLE.BIN/
-
-packages/
-.vs/
+
+packages/
+.vs/
+
+# Benchmark Dot Net
+**/BenchmarkDotNet.Artifacts/*
+**/project.lock.json
+tests/output/*
+.vs/restore.dg
+artifacts/*
+BDN.Generated
+BenchmarkDotNet.Samples/Properties/launchSettings.json
+src/BenchmarkDotNet/Disassemblers/net461/*

LazyCache.Benchmarks/BenchmarkConfig.cs

@@ -0,0 +1,22 @@
+using BenchmarkDotNet.Analysers;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
+using Perfolizer.Horology;
+
+namespace LazyCache.Benchmarks
+{
+    public class BenchmarkConfig: ManualConfig
+    {
+        public BenchmarkConfig()
+            => AddJob(Job.ShortRun)
+              .AddDiagnoser(MemoryDiagnoser.Default)
+              .AddLogger(new ConsoleLogger())
+              .AddColumn(TargetMethodColumn.Method)
+              .AddAnalyser(EnvironmentAnalyser.Default)
+              .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond));
+    }
+}

LazyCache.Benchmarks/ComplexObject.cs

@@ -0,0 +1,8 @@
+namespace LazyCache.Benchmarks
+{
+    public class ComplexObject
+    {
+        public string String { get; set; } = string.Empty;
+        public int Int { get; set; } = default;
+    }
+}

LazyCache.Benchmarks/LazyCache.Benchmarks.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <Optimize>true</Optimize>
+    <Configuration>Release</Configuration>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\LazyCache\LazyCache.csproj" />
+  </ItemGroup>
+
+</Project>

LazyCache.Benchmarks/MemoryCacheBenchmarks.cs

@@ -0,0 +1,92 @@
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using LazyCache.Providers;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace LazyCache.Benchmarks
+{
+    [Config(typeof(BenchmarkConfig))]
+    [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
+    public class MemoryCacheBenchmarks
+    { 
+        public const string CacheKey = nameof(CacheKey);
+
+        public IMemoryCache MemCache;
+        public IAppCache AppCache;
+        public ComplexObject ComplexObject;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            MemCache = new MemoryCache(new MemoryCacheOptions());
+            AppCache = new CachingService();
+
+            ComplexObject = new ComplexObject();
+        }
+
+        [GlobalCleanup]
+        public void Cleanup() => MemCache.Dispose();
+
+        /*
+         *
+         * Benchmark Cache Initialization
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory("Init")]
+        public MemoryCache DotNetMemoryCache_Init() => new MemoryCache(new MemoryCacheOptions());
+
+        [Benchmark, BenchmarkCategory("Init")]
+        public CachingService LazyCache_Init() => new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions())));
+
+        /*
+         *
+         * Benchmark Add Methods
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.Add))]
+        public void DotNetMemoryCache_Set() => MemCache.Set(CacheKey, ComplexObject);
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.Add))]
+        public void LazyCache_Set() => AppCache.Add(CacheKey, ComplexObject);
+
+        /*
+         *
+         * Benchmark Get Methods
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.Get))]
+        public ComplexObject DotNetMemoryCache_Get() => MemCache.Get<ComplexObject>(CacheKey);
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.Get))]
+        public ComplexObject LazyCache_Get() => AppCache.Get<ComplexObject>(CacheKey);
+
+        /*
+         *
+         * Benchmark GetOrAdd Methods
+         *
+         */
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAdd))]
+        public ComplexObject DotNetMemoryCache_GetOrAdd() => MemCache.GetOrCreate(CacheKey, entry => ComplexObject);
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAdd))]
+        public ComplexObject LazyCache_GetOrAdd() => AppCache.GetOrAdd(CacheKey, entry => ComplexObject);
+
+        /*
+         *
+         * Benchmark GetOrAddAsync Methods
+         *
+         */
+
+
+        [Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAddAsync))]
+        public async Task<ComplexObject> DotNetMemoryCache_GetOrAddAsync() => await MemCache.GetOrCreateAsync(CacheKey, async entry => await Task.FromResult(ComplexObject));
+
+        [Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAddAsync))]
+        public async Task<ComplexObject> LazyCache_GetOrAddAsync() => await AppCache.GetOrAddAsync(CacheKey, async entry => await Task.FromResult(ComplexObject));
+    }
+}

LazyCache.Benchmarks/MemoryCacheBenchmarksRealLifeScenarios.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Management;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Reports;
+using LazyCache.Providers;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace LazyCache.Benchmarks
+{
+    [Config(typeof(BenchmarkConfig))]
+    [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
+    public class MemoryCacheBenchmarksRealLifeScenarios
+    {
+        public const string CacheKey = nameof(CacheKey);
+
+        public ComplexObject ComplexObject1;
+        public ComplexObject ComplexObject2;
+        public ComplexObject ComplexObject3;
+        public ComplexObject ComplexObject4;
+        public ComplexObject ComplexObject5;
+
+        // Trying not to introduce artificial allocations below - just measuring what the library itself needs
+        [GlobalSetup]
+        public void Setup()
+        {
+            ComplexObject1 = new ComplexObject();
+            ComplexObject2 = new ComplexObject();
+            ComplexObject3 = new ComplexObject();
+            ComplexObject4 = new ComplexObject();
+            ComplexObject5 = new ComplexObject();
+        }
+
+        [Benchmark]
+        public ComplexObject Init_CRUD()
+        {
+            var cache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions()))) as IAppCache;
+
+            cache.Add(CacheKey, ComplexObject1);
+
+            var obj = cache.Get<ComplexObject>(CacheKey);
+
+            obj.Int = 256;
+            cache.Add(CacheKey, obj);
+
+            cache.Remove(CacheKey);
+
+            return obj;
+        }
+
+        // Benchmark memory usage to ensure only a single instance of the object is created
+        // Due to the nature of AsyncLazy, this test should also only take the the time it takes to create
+        //   one instance  of the object.
+        [Benchmark]
+        public async Task<byte[]> Several_initializations_of_1Mb_object_with_200ms_delay()
+        {
+            var cache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions()))) as IAppCache;
+
+            Task AddByteArrayToCache() =>
+                cache.GetOrAddAsync(CacheKey, async () =>
+                {
+                    await Task.Delay(200);
+                    return await Task.FromResult(new byte[1024 * 1024]); // 1Mb
+                });
+
+            // Even though the second and third init attempts are later, this whole operation should still take the time of the first
+            var creationTask1 = AddByteArrayToCache(); // initialization attempt, or 200ms
+            var creationTask2 = Delayed(50, AddByteArrayToCache);
+            var creationTask3 = Delayed(50, AddByteArrayToCache);
+
+            await Task.WhenAll(creationTask1, creationTask2, creationTask3);
+
+            return cache.Get<byte[]>(CacheKey);
+        }
+
+        private async Task Delayed(int ms, Func<Task> action)
+        {
+            await Task.Delay(ms);
+            await action();
+        }
+    }
+}

LazyCache.Benchmarks/Program.cs

@@ -0,0 +1,9 @@
+using BenchmarkDotNet.Running;
+
+namespace LazyCache.Benchmarks
+{
+    public static class Program
+    {
+        public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+    }
+}

LazyCache.Benchmarks/Properties/launchSettings.json

@@ -0,0 +1,7 @@
+{
+  "profiles": {
+    "LazyCache.Benchmarks": {
+      "commandName": "Project"
+    }
+  }
+}

LazyCache.Benchmarks/README.md

@@ -0,0 +1,64 @@
+# LazyCache.Benchmarks
+This project is dedicated towards benchmarking (using [BenchmarkDotNet](https://benchmarkdotnet.org/index.html)) the basic functionality of LazyCache such that contributors and maintainers can verify the efficacy of changes towards the project - for better or for worse.
+
+## Note to readers
+While it is always a good idea to understand performance of your third party libraries, it is rare that you will be concerned with performance on the scale of nanoseconds such that this library operates on. Be wary of premature optimization.
+
+# How to run
+- Ensure you have the requisite dotnet SDKs found in _LazyCache.Benchmarks.csproj_
+- Clone the project
+- Open your favorite terminal, navigate to the Benchmark Project
+- `dotnet run -c Release`
+- Pick your desired benchmark suite via numeric entry
+
+If you are interested in benchmarking a specific method (after making changes to it, for instance), you can conveniently filter down to one specific benchmark, e.g. `dotnet run -c Release -- -f *Get` will only run the benchmarks for `IAppCache.Get` implementations, likewise with `*GetOrAddAsync`, or other methods.
+
+# Contributing
+If you have ideas for one or more benchmarks not covered here, please add an issue describing what you would like to see. Pull requests are always welcome!
+
+# Benchmark Types
+There are two types of benchmarks available.
+
+## Basics
+The basic benchmarks are small and laser-focused on testing individual aspects of LazyCache. This suite of benchmarks uses the out-of-the-box MemoryCache from dotnet [seen here](https://github.com/dotnet/runtime/blob/master/src/libraries/Microsoft.Extensions.Caching.Memory/src/) as a baseline, to demonstrate the "cost" of LazyCache in comparison.
+
+## Integration
+These benchmarks are designed to showcase full use-cases of LazyCache by chaining together various operations. As an example, with the Memory Diagnoser from BenchmarkDotNet, we can verify that concurrent calls to initialize a cache item correctly spin up one instance of said item, with the subsequent calls awaiting its result.
+
+### Gotchas
+Remember that BenchmarkDotNet dutifully monitors allocations inside the benchmark method, and _only_ the method. At the time of writing, the default instance of the MemoryCacheProvider is static, and allocations into this cache will **not** be monitored by BenchmarkDotNet. For all benchmarks, please ensure you are creating new instances of the Service, Provider, and backing Cache.
+
+# Benchmarks
+
+```
+// * Summary *
+BenchmarkDotNet=v0.12.1, OS=Windows 10.0.18362.1082 (1903/May2019Update/19H1)
+AMD Ryzen 9 3900X, 1 CPU, 24 logical and 12 physical cores
+.NET Core SDK=5.0.100-preview.7.20366.6
+  [Host]   : .NET Core 3.1.7 (CoreCLR 4.700.20.36602, CoreFX 4.700.20.37001), X64 RyuJIT
+  ShortRun : .NET Core 3.1.7 (CoreCLR 4.700.20.36602, CoreFX 4.700.20.37001), X64 RyuJIT
+
+Job=ShortRun  IterationCount=3  LaunchCount=1
+WarmupCount=3
+```
+|                          Method |       Mean |     Error |   StdDev | Ratio | RatioSD |  Gen 0 |  Gen 1 |  Gen 2 | Allocated |
+|-------------------------------- |-----------:|----------:|---------:|------:|--------:|-------:|-------:|-------:|----------:|
+|          DotNetMemoryCache_Init | 1,605.6 ns | 221.54 ns | 12.14 ns |  1.00 |    0.00 | 0.1850 | 0.0916 | 0.0019 |    1560 B |
+|                  LazyCache_Init | 2,843.1 ns | 486.02 ns | 26.64 ns |  1.77 |    0.01 | 0.3090 | 0.1526 |      - |    2600 B |
+|                                 |            |           |          |       |         |        |        |        |           |
+|           DotNetMemoryCache_Set |   483.6 ns |   1.82 ns |  0.10 ns |  1.00 |    0.00 | 0.0496 |      - |      - |     416 B |
+|                   LazyCache_Set |   810.7 ns |   6.21 ns |  0.34 ns |  1.68 |    0.00 | 0.0801 |      - |      - |     672 B |
+|                                 |            |           |          |       |         |        |        |        |           |
+|           DotNetMemoryCache_Get |   197.8 ns |   5.49 ns |  0.30 ns |  1.00 |    0.00 |      - |      - |      - |         - |
+|                   LazyCache_Get |   231.3 ns |   3.25 ns |  0.18 ns |  1.17 |    0.00 |      - |      - |      - |         - |
+|                                 |            |           |          |       |         |        |        |        |           |
+|      DotNetMemoryCache_GetOrAdd |   260.6 ns |  18.44 ns |  1.01 ns |  1.00 |    0.00 | 0.0076 |      - |      - |      64 B |
+|              LazyCache_GetOrAdd |   370.1 ns |  30.55 ns |  1.67 ns |  1.42 |    0.01 | 0.0191 |      - |      - |     160 B |
+|                                 |            |           |          |       |         |        |        |        |           |
+| DotNetMemoryCache_GetOrAddAsync |   375.5 ns |  46.47 ns |  2.55 ns |  1.00 |    0.00 | 0.0334 |      - |      - |     280 B |
+|         LazyCache_GetOrAddAsync |   578.5 ns |  66.25 ns |  3.63 ns |  1.54 |    0.02 | 0.0534 |      - |      - |     448 B |
+
+|                                                 Method |             Mean |           Error |          StdDev |  Gen 0 |  Gen 1 |  Gen 2 |  Allocated |
+|------------------------------------------------------- |-----------------:|----------------:|----------------:|-------:|-------:|-------:|-----------:|
+|                                              Init_CRUD |       5,115.1 ns |        991.0 ns |        54.32 ns | 0.4730 | 0.2365 | 0.0076 |     3.9 KB |
+| Several_initializations_of_1Mb_object_with_200ms_delay | 207,329,988.9 ns | 31,342,899.9 ns | 1,718,010.11 ns |      - |      - |      - | 1031.75 KB |

LazyCache.sln

@@ -35,6 +35,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LazyCache.UnitTestsCore30",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LazyCache.UnitTestsCore31", "LazyCache.UnitTestsCore31\LazyCache.UnitTestsCore31.csproj", "{2E025606-884D-4C48-8490-99EB1EA7B268}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LazyCache.Benchmarks", "LazyCache.Benchmarks\LazyCache.Benchmarks.csproj", "{CE7DF61F-03B2-493E-8BFF-6E744015DE14}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -85,6 +87,10 @@ Global
 		{2E025606-884D-4C48-8490-99EB1EA7B268}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2E025606-884D-4C48-8490-99EB1EA7B268}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2E025606-884D-4C48-8490-99EB1EA7B268}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CE7DF61F-03B2-493E-8BFF-6E744015DE14}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CE7DF61F-03B2-493E-8BFF-6E744015DE14}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CE7DF61F-03B2-493E-8BFF-6E744015DE14}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CE7DF61F-03B2-493E-8BFF-6E744015DE14}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE