Add benchmarks for flag evaluation

configcat · Aug 4, 2023 · cd4b022 · cd4b022
1 parent d794cc2
commit cd4b022
Show file tree

Hide file tree

Showing 24 changed files with 1,115 additions and 217 deletions.
diff --git a/.gitignore b/.gitignore
@@ -288,6 +288,7 @@ __pycache__/
 *.xsd.cs
 
 # Custom
+BenchmarkDotNet.Artifacts*/
 coverage.xml
 .DS_Store
 .vscode

diff --git a/benchmarks/ConfigCat.Client.Benchmarks.sln b/benchmarks/ConfigCat.Client.Benchmarks.sln
@@ -0,0 +1,60 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.6.33815.320
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ConfigCat.Client.Benchmarks", "ConfigCat.Client.Benchmarks\ConfigCat.Client.Benchmarks.csproj", "{B7381881-0709-4F72-AE6C-3778979CD8C1}"
+	ProjectSection(ProjectDependencies) = postProject
+		{B51439A6-F230-46E5-9BC3-7E4E9FA841FC} = {B51439A6-F230-46E5-9BC3-7E4E9FA841FC}
+	EndProjectSection
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ConfigCatClient", "..\src\ConfigCatClient\ConfigCatClient.csproj", "{B51439A6-F230-46E5-9BC3-7E4E9FA841FC}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OldVersionLib", "OldVersionLib\OldVersionLib.csproj", "{53015044-8ED1-4F77-BB02-357313F7952A}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NewVersionLib", "NewVersionLib\NewVersionLib.csproj", "{30B22E19-6701-4C36-B1F4-72AE24E93CEA}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ProjectReferences", "ProjectReferences", "{3B9B9CF8-8D20-423D-A327-60D2D2C77976}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Benchmark|Any CPU = Benchmark|Any CPU
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{B7381881-0709-4F72-AE6C-3778979CD8C1}.Benchmark|Any CPU.ActiveCfg = Release|Any CPU
+		{B7381881-0709-4F72-AE6C-3778979CD8C1}.Benchmark|Any CPU.Build.0 = Release|Any CPU
+		{B7381881-0709-4F72-AE6C-3778979CD8C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B7381881-0709-4F72-AE6C-3778979CD8C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B7381881-0709-4F72-AE6C-3778979CD8C1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B7381881-0709-4F72-AE6C-3778979CD8C1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B51439A6-F230-46E5-9BC3-7E4E9FA841FC}.Benchmark|Any CPU.ActiveCfg = Benchmark|Any CPU
+		{B51439A6-F230-46E5-9BC3-7E4E9FA841FC}.Benchmark|Any CPU.Build.0 = Benchmark|Any CPU
+		{B51439A6-F230-46E5-9BC3-7E4E9FA841FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B51439A6-F230-46E5-9BC3-7E4E9FA841FC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B51439A6-F230-46E5-9BC3-7E4E9FA841FC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B51439A6-F230-46E5-9BC3-7E4E9FA841FC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{53015044-8ED1-4F77-BB02-357313F7952A}.Benchmark|Any CPU.ActiveCfg = Release|Any CPU
+		{53015044-8ED1-4F77-BB02-357313F7952A}.Benchmark|Any CPU.Build.0 = Release|Any CPU
+		{53015044-8ED1-4F77-BB02-357313F7952A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{53015044-8ED1-4F77-BB02-357313F7952A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{53015044-8ED1-4F77-BB02-357313F7952A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{53015044-8ED1-4F77-BB02-357313F7952A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{30B22E19-6701-4C36-B1F4-72AE24E93CEA}.Benchmark|Any CPU.ActiveCfg = Release|Any CPU
+		{30B22E19-6701-4C36-B1F4-72AE24E93CEA}.Benchmark|Any CPU.Build.0 = Release|Any CPU
+		{30B22E19-6701-4C36-B1F4-72AE24E93CEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{30B22E19-6701-4C36-B1F4-72AE24E93CEA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{30B22E19-6701-4C36-B1F4-72AE24E93CEA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{30B22E19-6701-4C36-B1F4-72AE24E93CEA}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{B51439A6-F230-46E5-9BC3-7E4E9FA841FC} = {3B9B9CF8-8D20-423D-A327-60D2D2C77976}
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {71FC06CD-80AD-4090-863E-1965313C9027}
+	EndGlobalSection
+EndGlobal
diff --git a/benchmarks/ConfigCat.Client.Benchmarks/ConfigCat.Client.Benchmarks.csproj b/benchmarks/ConfigCat.Client.Benchmarks/ConfigCat.Client.Benchmarks.csproj
@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net48;net6.0</TargetFrameworks>
+    <LangVersion>10.0</LangVersion>
+    <Nullable>enable</Nullable>
+    <WarningsAsErrors>nullable</WarningsAsErrors>
+    <SignAssembly>true</SignAssembly>
+    <AssemblyOriginatorKeyFile>..\..\src\ConfigCatClient.snk</AssemblyOriginatorKeyFile>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.5" />
+    <PackageReference Include="ConfigCat.Client" Version="8.2.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\ConfigCatClient\ConfigCatClient.csproj">
+      <AdditionalProperties>Configuration=Benchmark</AdditionalProperties>
+      <Aliases>from_project</Aliases>
+    </ProjectReference>
+    <ProjectReference Include="..\NewVersionLib\NewVersionLib.csproj" />
+    <ProjectReference Include="..\OldVersionLib\OldVersionLib.csproj" />
+  </ItemGroup>
+
+  <Target Name="ChangeAliasOfReferencedNuget" BeforeTargets="FindReferenceAssembliesForReferences;ResolveReferences">
+    <ItemGroup>
+      <ReferencePath Condition="'%(FileName)' == 'ConfigCat.Client'">
+        <Aliases>from_nuget</Aliases>
+      </ReferencePath>
+    </ItemGroup>
+  </Target>
+
+</Project>
diff --git a/benchmarks/ConfigCat.Client.Benchmarks/FlagEvaluationBenchmark.cs b/benchmarks/ConfigCat.Client.Benchmarks/FlagEvaluationBenchmark.cs
@@ -0,0 +1,68 @@
+extern alias from_nuget;
+extern alias from_project;
+
+using System;
+using BenchmarkDotNet.Attributes;
+
+namespace ConfigCat.Client.Benchmarks;
+
+[MemoryDiagnoser]
+public class FlagEvaluationBenchmark
+{
+    private object evaluationServicesOld = null!;
+    private from_nuget::ConfigCat.Client.User userOld = null!;
+
+    private object evaluationServicesNew = null!;
+    private from_project::ConfigCat.Client.User userNew = null!;
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        Environment.CurrentDirectory = AppContext.BaseDirectory;
+
+        this.evaluationServicesOld = Old.BenchmarkHelper.CreateEvaluationServices(LogInfo);
+        this.userOld = new("Cat") { Email = "cat@configcat.com", Custom = { ["Version"] = "1.1.1", ["Number"] = "1" } };
+
+        this.evaluationServicesNew = New.BenchmarkHelper.CreateEvaluationServices(LogInfo);
+        this.userNew = new("Cat") { Email = "cat@configcat.com", Custom = { ["Version"] = "1.1.1", ["Number"] = "1" } };
+    }
+
+    [Params(false, true)]
+    public bool LogInfo { get; set; }
+
+    [Benchmark]
+    public object Basic_ConfigV5()
+    {
+        return Old.BenchmarkHelper.Evaluate(this.evaluationServicesOld, "basicFlag", false);
+    }
+
+    [Benchmark]
+    public object Basic_ConfigV6()
+    {
+        return New.BenchmarkHelper.Evaluate(this.evaluationServicesNew, "basicFlag", false);
+    }
+
+    [Benchmark]
+    public object Complex_ConfigV5()
+    {
+        return Old.BenchmarkHelper.Evaluate(this.evaluationServicesOld, "complexFlag", "", this.userOld);
+    }
+
+    [Benchmark]
+    public object Complex_ConfigV6()
+    {
+        return New.BenchmarkHelper.Evaluate(this.evaluationServicesNew, "complexFlag", "", this.userNew);
+    }
+
+    [Benchmark]
+    public object All_ConfigV5()
+    {
+        return Old.BenchmarkHelper.EvaluateAll(this.evaluationServicesOld, this.userOld);
+    }
+
+    [Benchmark]
+    public object All_ConfigV6()
+    {
+        return New.BenchmarkHelper.EvaluateAll(this.evaluationServicesNew, this.userNew);
+    }
+}
diff --git a/benchmarks/JsonDeserializationBenchmark.cs → ...enchmarks/JsonDeserializationBenchmark.cs b/benchmarks/JsonDeserializationBenchmark.cs → ...enchmarks/JsonDeserializationBenchmark.cs
@@ -1,24 +1,24 @@
-extern alias from_nuget;
+extern alias from_nuget;
 extern alias from_project;
 using BenchmarkDotNet.Attributes;
 using System;
 
-namespace ConfigCatClient.Benchmarks;
+namespace ConfigCat.Client.Benchmarks;
 
 [MemoryDiagnoser]
 public class JsonDeserializationBenchmark
 {
-    private readonly from_project::ConfigCat.Client.IConfigCatClient newClient = from_project::ConfigCat.Client.ConfigCatClientBuilder
-        .Initialize("rv3YCMKenkaM7xkOCVQfeg/-I_w49WSQUWdZypPPM4Yyg")
-        .WithManualPoll()
-        .WithBaseUrl(new Uri("https://test-cdn-global.configcat.com"))
-        .Create();
+    private readonly from_project::ConfigCat.Client.IConfigCatClient newClient = from_project::ConfigCat.Client.ConfigCatClient.Get("rv3YCMKenkaM7xkOCVQfeg/-I_w49WSQUWdZypPPM4Yyg", o =>
+    {
+        o.PollingMode = from_project::ConfigCat.Client.PollingModes.ManualPoll;
+        o.BaseUrl = new Uri("https://test-cdn-global.configcat.com");
+    });
 
-    private readonly from_nuget::ConfigCat.Client.IConfigCatClient oldClient = from_nuget::ConfigCat.Client.ConfigCatClientBuilder
-        .Initialize("rv3YCMKenkaM7xkOCVQfeg/-I_w49WSQUWdZypPPM4Yyg")
-        .WithManualPoll()
-        .WithBaseUrl(new Uri("https://test-cdn-global.configcat.com"))
-        .Create();
+    private readonly from_nuget::ConfigCat.Client.IConfigCatClient oldClient = from_nuget::ConfigCat.Client.ConfigCatClient.Get("rv3YCMKenkaM7xkOCVQfeg/-I_w49WSQUWdZypPPM4Yyg", o =>
+    {
+        o.PollingMode = from_nuget::ConfigCat.Client.PollingModes.ManualPoll;
+        o.BaseUrl = new Uri("https://test-cdn-global.configcat.com");
+    });
 
     private readonly from_project::ConfigCat.Client.User newUser = new("test@test.com");
     private readonly from_nuget::ConfigCat.Client.User oldUser = new("test@test.com");

diff --git a/benchmarks/ConfigCat.Client.Benchmarks/MatrixTestBenchmark.cs b/benchmarks/ConfigCat.Client.Benchmarks/MatrixTestBenchmark.cs
@@ -0,0 +1,45 @@
+using System;
+using BenchmarkDotNet.Attributes;
+
+namespace ConfigCat.Client.Benchmarks;
+
+[MemoryDiagnoser]
+public class MatrixTestBenchmark
+{
+    private Old.MatrixTestRunnerBase<Old.BenchmarkHelper.BasicMatrixTestsDescriptor> testRunnerOld = null!;
+    private object evaluationServicesOld = null!;
+
+    private New.MatrixTestRunnerBase<New.BenchmarkHelper.BasicMatrixTestsDescriptor> testRunnerNew = null!;
+    private object evaluationServicesNew = null!;
+
+    private object?[][] tests = null!;
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        Environment.CurrentDirectory = AppContext.BaseDirectory;
+
+        this.testRunnerOld = new();
+        this.evaluationServicesOld = Old.BenchmarkHelper.CreateEvaluationServices(LogInfo);
+
+        this.testRunnerNew = new();
+        this.evaluationServicesNew = New.BenchmarkHelper.CreateEvaluationServices(LogInfo);
+
+        this.tests = New.BenchmarkHelper.GetMatrixTests<New.BenchmarkHelper.BasicMatrixTestsDescriptor>();
+    }
+
+    [Params(false, true)]
+    public bool LogInfo { get; set; }
+
+    [Benchmark]
+    public int MatrixTests_ConfigV5()
+    {
+        return Old.BenchmarkHelper.RunAllMatrixTests(this.testRunnerOld, this.evaluationServicesOld, this.tests);
+    }
+
+    [Benchmark]
+    public int MatrixTests_ConfigV6()
+    {
+        return New.BenchmarkHelper.RunAllMatrixTests(this.testRunnerNew, this.evaluationServicesNew, this.tests);
+    }
+}
diff --git a/benchmarks/ConfigCat.Client.Benchmarks/Program.cs b/benchmarks/ConfigCat.Client.Benchmarks/Program.cs
@@ -0,0 +1,11 @@
+using BenchmarkDotNet.Running;
+
+namespace ConfigCatClient.Benchmarks;
+
+internal class Program
+{
+    private static void Main(string[] args)
+    {
+        BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+    }
+}
diff --git a/benchmarks/ConfigCatClient.Benchmarks.csproj b/benchmarks/ConfigCatClient.Benchmarks.csproj
diff --git a/benchmarks/ConfigCatClient.Benchmarks.sln b/benchmarks/ConfigCatClient.Benchmarks.sln