Fixed problems with benchmarks and adds reference results (#4659)

ChilliCream · Jan 18, 2022 · f3a4011 · f3a4011
1 parent d199ca7
commit f3a4011
Show file tree

Hide file tree

Showing 17 changed files with 206 additions and 15 deletions.
diff --git a/src/HotChocolate/Benchmarks/execution.cmd b/src/HotChocolate/Benchmarks/execution.cmd
@@ -0,0 +1,24 @@
+@echo off
+
+pushd "%~dp0"
+
+if exist BenchmarkDotNet.Artifacts (
+	rmdir /s /q BenchmarkDotNet.Artifacts
+)
+if exist src\Benchmarks.Execution\bin (
+	rmdir /s /q src\Benchmarks.Execution\bin
+)
+if exist src\Benchmarks.Execution\obj (
+	rmdir /s /q src\Benchmarks.Execution\obj
+)
+
+dotnet run --project src\Benchmarks.Execution\HotChocolate.Benchmarks.Execution.csproj -c release --filter "HotChocolate.Benchmarks.*"
+
+if exist src\Benchmarks.Execution\bin (
+	rmdir /s /q src\Benchmarks.Execution\bin
+)
+if exist src\Benchmarks.Execution\obj (
+	rmdir /s /q src\Benchmarks.Execution\obj
+)
+
+popd
diff --git a/src/HotChocolate/Benchmarks/results/net6/mta/ExtensionDataBenchmarks.md b/src/HotChocolate/Benchmarks/results/net6/mta/ExtensionDataBenchmarks.md
@@ -0,0 +1,22 @@
+``` ini
+
+BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19044.1466 (21H2)
+11th Gen Intel Core i7-11700F 2.50GHz, 1 CPU, 16 logical and 8 physical cores
+.NET SDK=6.0.101
+  [Host]     : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+  DefaultJob : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+
+
+```
+|                        Method |         Mean |     Error |    StdDev |       Median | Rank |  Gen 0 |  Gen 1 | Allocated |
+|------------------------------ |-------------:|----------:|----------:|-------------:|-----:|-------:|-------:|----------:|
+|          Create_ExtensionData |     2.589 ns | 0.0271 ns | 0.0226 ns |     2.595 ns |    2 | 0.0029 |      - |      24 B |
+|        Create_ExtensionData_2 |     1.707 ns | 0.0227 ns | 0.0212 ns |     1.706 ns |    1 | 0.0029 |      - |      24 B |
+|    Create_ExtensionData_Set_1 |    72.679 ns | 0.6837 ns | 0.6396 ns |    72.257 ns |    7 | 0.0153 |      - |     128 B |
+|  Create_ExtensionData_2_Set_1 |    29.368 ns | 0.1376 ns | 0.1149 ns |    29.363 ns |    5 | 0.0287 |      - |     240 B |
+|    Create_ExtensionData_Set_2 |   187.498 ns | 1.6950 ns | 1.5026 ns |   187.719 ns |    8 | 0.0410 |      - |     344 B |
+|  Create_ExtensionData_2_Set_2 |    46.798 ns | 0.3243 ns | 0.3033 ns |    46.856 ns |    6 | 0.0344 |      - |     288 B |
+|   Create_ExtensionData_Set_10 | 1,535.841 ns | 4.6301 ns | 4.3310 ns | 1,535.178 ns |   10 | 0.3147 | 0.0019 |   2,648 B |
+| Create_ExtensionData_2_Set_10 |   229.478 ns | 1.1805 ns | 1.1042 ns |   229.666 ns |    9 | 0.1500 | 0.0010 |   1,256 B |
+|    Create_ExtensionData_Get_1 |    11.472 ns | 0.0371 ns | 0.0329 ns |    11.476 ns |    4 |      - |      - |         - |
+|  Create_ExtensionData_2_Get_1 |     8.100 ns | 0.0167 ns | 0.0140 ns |     8.099 ns |    3 |      - |      - |         - |
diff --git a/src/HotChocolate/Benchmarks/results/net6/mta/HttpQueryBenchmarks.md b/src/HotChocolate/Benchmarks/results/net6/mta/HttpQueryBenchmarks.md
@@ -0,0 +1,16 @@
+``` ini
+
+BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19044.1466 (21H2)
+11th Gen Intel Core i7-11700F 2.50GHz, 1 CPU, 16 logical and 8 physical cores
+.NET SDK=6.0.101
+  [Host]     : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+  DefaultJob : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+
+
+```
+|                                         Method |       Mean |    Error |   StdDev |     Median | Rank |    Gen 0 |    Gen 1 |   Gen 2 | Allocated |
+|----------------------------------------------- |-----------:|---------:|---------:|-----------:|-----:|---------:|---------:|--------:|----------:|
+|             Sessions_TitleAndAbstract_10_Items |   255.8 μs |  0.89 μs |  0.79 μs |   255.9 μs |    1 |  10.2539 |   1.9531 |       - |     83 KB |
+| Sessions_TitleAndAbstractAndTrackName_10_Items |   458.3 μs |  4.03 μs |  3.77 μs |   458.0 μs |    2 |  15.6250 |   3.9063 |       - |    128 KB |
+|                                Sessions_Medium | 1,982.5 μs | 15.87 μs | 14.07 μs | 1,983.4 μs |    3 | 125.0000 |  46.8750 |  7.8125 |  1,076 KB |
+|                                 Sessions_Large | 5,265.9 μs | 39.77 μs | 37.20 μs | 5,281.7 μs |    4 | 273.4375 | 125.0000 | 23.4375 |  2,287 KB |
diff --git a/src/HotChocolate/Benchmarks/results/net6/mta/IntrospectionBenchmarks.md b/src/HotChocolate/Benchmarks/results/net6/mta/IntrospectionBenchmarks.md
@@ -0,0 +1,15 @@
+``` ini
+
+BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19044.1466 (21H2)
+11th Gen Intel Core i7-11700F 2.50GHz, 1 CPU, 16 logical and 8 physical cores
+.NET SDK=6.0.101
+  [Host]     : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+  DefaultJob : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+
+
+```
+|                       Method |       Mean |     Error |    StdDev |     Median | Rank |   Gen 0 |  Gen 1 | Allocated |
+|----------------------------- |-----------:|----------:|----------:|-----------:|-----:|--------:|-------:|----------:|
+|               Query_TypeName |   4.766 μs | 0.0306 μs | 0.0286 μs |   4.767 μs |    1 |  0.2594 |      - |      2 KB |
+|          Query_Introspection | 730.073 μs | 6.9186 μs | 6.4717 μs | 730.246 μs |    3 | 27.3438 | 0.9766 |    230 KB |
+| Query_Introspection_Prepared | 673.435 μs | 3.3553 μs | 2.9744 μs | 672.392 μs |    2 | 27.3438 | 0.9766 |    229 KB |
diff --git a/src/HotChocolate/Benchmarks/results/net6/mta/QueryBenchmarks.md b/src/HotChocolate/Benchmarks/results/net6/mta/QueryBenchmarks.md
@@ -0,0 +1,17 @@
+``` ini
+
+BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19044.1466 (21H2)
+11th Gen Intel Core i7-11700F 2.50GHz, 1 CPU, 16 logical and 8 physical cores
+.NET SDK=6.0.101
+  [Host]     : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+  DefaultJob : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+
+
+```
+|                                         Method |        Mean |    Error |   StdDev |      Median | Rank |    Gen 0 |    Gen 1 | Allocated |
+|----------------------------------------------- |------------:|---------:|---------:|------------:|-----:|---------:|---------:|----------:|
+|             Sessions_TitleAndAbstract_10_Items |    140.1 μs |  0.39 μs |  0.35 μs |    140.1 μs |    1 |   5.8594 |   1.4648 |     50 KB |
+| Sessions_TitleAndAbstractAndTrackName_10_Items |    349.4 μs |  1.30 μs |  1.22 μs |    349.4 μs |    2 |  11.2305 |   2.4414 |     93 KB |
+|                                Sessions_Medium |  1,633.8 μs |  7.34 μs |  6.87 μs |  1,634.9 μs |    3 |  91.7969 |  35.1563 |    747 KB |
+|                                 Sessions_Large |  4,424.0 μs | 15.91 μs | 14.88 μs |  4,428.2 μs |    4 | 218.7500 |  70.3125 |  1,790 KB |
+|                      Sessions_DataLoader_Large | 11,618.3 μs | 50.63 μs | 47.36 μs | 11,632.4 μs |    5 | 437.5000 | 187.5000 |  3,670 KB |
diff --git a/src/HotChocolate/Benchmarks/results/net6/mta/SchemaBuildingBenchmarks.md b/src/HotChocolate/Benchmarks/results/net6/mta/SchemaBuildingBenchmarks.md
@@ -0,0 +1,15 @@
+``` ini
+
+BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19044.1466 (21H2)
+11th Gen Intel Core i7-11700F 2.50GHz, 1 CPU, 16 logical and 8 physical cores
+.NET SDK=6.0.101
+  [Host]     : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+  DefaultJob : .NET 6.0.1 (6.0.121.56705), X64 RyuJIT
+
+
+```
+|                              Method |      Mean |    Error |   StdDev |    Median | Rank |      Gen 0 |     Gen 1 |     Gen 2 | Allocated |
+|------------------------------------ |----------:|---------:|---------:|----------:|-----:|-----------:|----------:|----------:|----------:|
+| CreateSchema_AnnotationBased_Medium |  63.40 ms | 0.220 ms | 0.183 ms |  63.38 ms |    2 |  2000.0000 |  857.1429 |         - |     16 MB |
+|     CreateSchema_SchemaFirst_Medium | 263.13 ms | 2.883 ms | 2.697 ms | 262.93 ms |    3 | 15000.0000 | 5000.0000 | 2000.0000 |    126 MB |
+|      CreateSchema_SchemaFirst_Large |  11.53 ms | 0.049 ms | 0.046 ms |  11.53 ms |    1 |   781.2500 |  375.0000 |         - |      6 MB |
diff --git a/src/HotChocolate/Core/benchmark/Execution.Benchmarks/DefaultExecutionPipelineBenchmark.cs b/src/HotChocolate/Core/benchmark/Execution.Benchmarks/DefaultExecutionPipelineBenchmark.cs
@@ -133,10 +133,8 @@ private static async Task OneRequest(
 
             if (result.Errors is { Count: > 0 })
             {
-                await File.AppendAllTextAsync(
-                    "/Users/michael/local/hc-1/src/HotChocolate/Core/benchmark/err.log",
-                    result.ToJson());
-
+                Console.WriteLine("Full Error:");
+                Console.WriteLine(result.ToJson());
                 throw new InvalidOperationException(result.Errors[0].Message);
             }
 

diff --git a/...otChocolate/Core/benchmark/Execution.Benchmarks/Resources/GetHeroWithFriendsQuery.graphql b/...otChocolate/Core/benchmark/Execution.Benchmarks/Resources/GetHeroWithFriendsQuery.graphql
@@ -21,7 +21,7 @@ fragment Droid on Droid {
   height
 }
 
-fragment Friend on CharacterConnection {
+fragment Friend on FriendsConnection {
   nodes {
     ...HasName
     friends {

diff --git a/...late/Core/benchmark/Execution.Benchmarks/Resources/GetTwoHeroesWithFriendsRequest.graphql b/...late/Core/benchmark/Execution.Benchmarks/Resources/GetTwoHeroesWithFriendsRequest.graphql
@@ -24,7 +24,7 @@ fragment Droid on Droid {
   height
 }
 
-fragment Friend on CharacterConnection {
+fragment Friend on FriendsConnection {
   nodes {
     ...HasName
     friends {

diff --git a/src/HotChocolate/Core/benchmark/Execution.Benchmarks/Resources/LargeQuery.graphql b/src/HotChocolate/Core/benchmark/Execution.Benchmarks/Resources/LargeQuery.graphql
@@ -57,7 +57,7 @@ fragment Droid on Droid {
   height
 }
 
-fragment Friend on CharacterConnection {
+fragment Friend on FriendsConnection {
   nodes {
     ...HasName
     friends {

diff --git a/src/HotChocolate/Core/benchmark/Validation.Benchmarks/Resources/StarWarsQuery.graphql b/src/HotChocolate/Core/benchmark/Validation.Benchmarks/Resources/StarWarsQuery.graphql
@@ -51,7 +51,7 @@ fragment Starship on Starship {
   name
 }
 
-fragment Friend on CharacterConnection {
+fragment Friend on FriendsConnection {
   nodes {
     ...HasName
     friends {

diff --git a/src/HotChocolate/Core/benchmark/Validation.Benchmarks/ValidationBenchmarks.cs b/src/HotChocolate/Core/benchmark/Validation.Benchmarks/ValidationBenchmarks.cs
@@ -3,6 +3,7 @@
 using BenchmarkDotNet.Attributes;
 using HotChocolate.Language;
 using HotChocolate.StarWars;
+using HotChocolate.Execution;
 
 namespace HotChocolate.Validation.Benchmarks
 {
@@ -28,7 +29,9 @@ public ValidationBenchmarks()
             var factory = _services.GetRequiredService<IDocumentValidatorFactory>();
             _validator = factory.CreateValidator();
 
-            _schema = _services.GetRequiredService<ISchema>();
+            _schema = _services.GetRequiredService<IRequestExecutorResolver>()
+                .GetRequestExecutorAsync()
+                .Result.Schema;
             var resources = new ResourceHelper();
             _introspectionQuery = Utf8GraphQLParser.Parse(
                 resources.GetResourceString("IntrospectionQuery.graphql"));

diff --git a/src/HotChocolate/Core/benchmark/execution.cmd b/src/HotChocolate/Core/benchmark/execution.cmd
@@ -0,0 +1,24 @@
+@echo off
+
+pushd "%~dp0"
+
+if exist BenchmarkDotNet.Artifacts (
+	rmdir /s /q BenchmarkDotNet.Artifacts
+)
+if exist Execution.Benchmarks\bin (
+	rmdir /s /q Execution.Benchmarks\bin
+)
+if exist Execution.Benchmarks\obj (
+	rmdir /s /q Execution.Benchmarks\obj
+)
+
+dotnet run --project Execution.Benchmarks\HotChocolate.Execution.Benchmarks.csproj -c release
+
+if exist Execution.Benchmarks\bin (
+	rmdir /s /q Execution.Benchmarks\bin
+)
+if exist Execution.Benchmarks\obj (
+	rmdir /s /q Execution.Benchmarks\obj
+)
+
+popd
diff --git a/...tChocolate/Core/benchmark/results/net6/mta/DefaultExecutionPipelineBenchmark.md b/...tChocolate/Core/benchmark/results/net6/mta/DefaultExecutionPipelineBenchmark.md
@@ -0,0 +1,22 @@
+``` ini
+
+BenchmarkDotNet=v0.12.1, OS=Windows 10.0.19044
+11th Gen Intel Core i7-11700F 2.50GHz, 1 CPU, 16 logical and 8 physical cores
+.NET Core SDK=6.0.101
+  [Host]     : .NET Core 6.0.1 (CoreCLR 6.0.121.56705, CoreFX 6.0.121.56705), X64 RyuJIT
+  DefaultJob : .NET Core 6.0.1 (CoreCLR 6.0.121.56705, CoreFX 6.0.121.56705), X64 RyuJIT
+
+
+```
+|                                      Method |        Mean |       Error |      StdDev |      Median | Rank |     Gen 0 |    Gen 1 | Gen 2 |   Allocated |
+|-------------------------------------------- |------------:|------------:|------------:|------------:|-----:|----------:|---------:|------:|------------:|
+|                         SchemaIntrospection |    242.3 μs |     0.87 μs |     0.82 μs |    242.1 μs |    1 |   10.2539 |        - |     - |    84.12 KB |
+|     SchemaIntrospectionFiveParallelRequests |  1,219.7 μs |     9.00 μs |     8.42 μs |  1,218.8 μs |    2 |   50.7813 |        - |     - |   420.59 KB |
+|                                     GetHero | 15,562.3 μs |   248.69 μs |   232.62 μs | 15,702.1 μs |    3 |         - |        - |     - |     5.58 KB |
+|                 GetHeroFiveParallelRequests | 15,590.7 μs |   207.16 μs |   193.78 μs | 15,693.9 μs |    3 |         - |        - |     - |    27.68 KB |
+|                          GetHeroWithFriends | 46,728.6 μs |   610.80 μs |   541.46 μs | 46,978.4 μs |    4 |         - |        - |     - |    51.14 KB |
+|      GetHeroWithFriendsFiveParallelRequests | 46,902.5 μs |   192.30 μs |   150.13 μs | 46,944.4 μs |    4 |         - |        - |     - |   256.59 KB |
+|                      GetTwoHerosWithFriends | 46,873.5 μs |   448.68 μs |   397.74 μs | 46,985.8 μs |    4 |         - |        - |     - |   109.06 KB |
+| GetTwoHeroesWithFriendsFiveParallelRequests | 47,202.2 μs |   753.44 μs |   704.77 μs | 46,887.3 μs |    4 |         - |        - |     - |   529.34 KB |
+|                                  LargeQuery | 64,337.4 μs | 1,276.28 μs | 1,987.02 μs | 64,146.8 μs |    5 |  125.0000 |        - |     - |  1914.96 KB |
+|              LargeQueryFiveParallelRequests | 73,617.7 μs | 1,814.20 μs | 5,349.20 μs | 73,850.1 μs |    6 | 1142.8571 | 571.4286 |     - | 10218.22 KB |
diff --git a/src/HotChocolate/Core/benchmark/results/net6/mta/ValidationBenchmarks.md b/src/HotChocolate/Core/benchmark/results/net6/mta/ValidationBenchmarks.md
@@ -0,0 +1,14 @@
+``` ini
+
+BenchmarkDotNet=v0.12.1, OS=Windows 10.0.19044
+11th Gen Intel Core i7-11700F 2.50GHz, 1 CPU, 16 logical and 8 physical cores
+.NET Core SDK=6.0.101
+  [Host]     : .NET Core 6.0.1 (CoreCLR 6.0.121.56705, CoreFX 6.0.121.56705), X64 RyuJIT
+  DefaultJob : .NET Core 6.0.1 (CoreCLR 6.0.121.56705, CoreFX 6.0.121.56705), X64 RyuJIT
+
+
+```
+|                Method |     Mean |   Error |  StdDev |   Median | Rank |  Gen 0 | Gen 1 | Gen 2 | Allocated |
+|---------------------- |---------:|--------:|--------:|---------:|-----:|-------:|------:|------:|----------:|
+| ValidateIntrospection | 118.3 μs | 1.25 μs | 1.10 μs | 118.6 μs |    2 | 0.2441 |     - |     - |   2.54 KB |
+|      ValidateStarWars | 115.2 μs | 0.27 μs | 0.26 μs | 115.2 μs |    1 | 0.2441 |     - |     - |   2.54 KB |
diff --git a/src/HotChocolate/Core/benchmark/validation.cmd b/src/HotChocolate/Core/benchmark/validation.cmd
@@ -0,0 +1,24 @@
+@echo off
+
+pushd "%~dp0"
+
+if exist BenchmarkDotNet.Artifacts (
+	rmdir /s /q BenchmarkDotNet.Artifacts
+)
+if exist Validation.Benchmarks\bin (
+	rmdir /s /q Validation.Benchmarks\bin
+)
+if exist Validation.Benchmarks\obj (
+	rmdir /s /q Validation.Benchmarks\obj
+)
+
+dotnet run --project Validation.Benchmarks\HotChocolate.Validation.Benchmarks.csproj -c release
+
+if exist Validation.Benchmarks\bin (
+	rmdir /s /q Validation.Benchmarks\bin
+)
+if exist Validation.Benchmarks\obj (
+	rmdir /s /q Validation.Benchmarks\obj
+)
+
+popd
diff --git a/src/HotChocolate/Core/benchmark10/Execution.Benchmarks/DefaultExecutionPipelineBenchmark.cs b/src/HotChocolate/Core/benchmark10/Execution.Benchmarks/DefaultExecutionPipelineBenchmark.cs
@@ -131,12 +131,9 @@ private static async Task<IExecutionResult> OneRequest(
             }
             catch (Exception ex)
             {
-                var err = new StringBuilder();
-                err.AppendLine(ex.Message);
-                err.AppendLine(ex.StackTrace);
-                await File.AppendAllTextAsync(
-                    "/Users/michael/local/hc-1/src/HotChocolate/Core/benchmark/err.log",
-                    err.ToString());
+                Console.WriteLine("Full Exception:");
+                Console.WriteLine(ex.Message);
+                Console.WriteLine(ex.StackTrace);
                 throw;
             }
         }