From 405dfc585e633e3bf9222ed28348c325e8d1204e Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Wed, 26 Jan 2022 04:48:09 +0700 Subject: [PATCH] Add CPU benchmark project (#5536) * Add CPU benchmark project * Fix typos * Add README.md Co-authored-by: Aaron Stannard --- src/Akka.sln | 15 ++ .../Akka.Cluster.Cpu.Benchmark.csproj | 18 ++ .../BenchmarkNode.cs | 71 +++++++ .../Akka.Cluster.Cpu.Benchmark/Program.cs | 197 ++++++++++++++++++ .../Akka.Cluster.Cpu.Benchmark/README.md | 36 ++++ 5 files changed, 337 insertions(+) create mode 100644 src/benchmark/Akka.Cluster.Cpu.Benchmark/Akka.Cluster.Cpu.Benchmark.csproj create mode 100644 src/benchmark/Akka.Cluster.Cpu.Benchmark/BenchmarkNode.cs create mode 100644 src/benchmark/Akka.Cluster.Cpu.Benchmark/Program.cs create mode 100644 src/benchmark/Akka.Cluster.Cpu.Benchmark/README.md diff --git a/src/Akka.sln b/src/Akka.sln index 53eb1535097..f41238272c1 100644 --- a/src/Akka.sln +++ b/src/Akka.sln @@ -256,6 +256,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Akka.Persistence.Custom", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Akka.Persistence.Custom.Tests", "examples\Akka.Persistence.Custom.Tests\Akka.Persistence.Custom.Tests.csproj", "{F6C974B8-48F8-41C7-95AC-3CFAA720E0E4}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Akka.Cluster.Cpu.Benchmark", "benchmark\Akka.Cluster.Cpu.Benchmark\Akka.Cluster.Cpu.Benchmark.csproj", "{6FA94D22-9369-4A60-BBC1-764CA68F4ED1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1195,6 +1197,18 @@ Global {F6C974B8-48F8-41C7-95AC-3CFAA720E0E4}.Release|x64.Build.0 = Release|Any CPU {F6C974B8-48F8-41C7-95AC-3CFAA720E0E4}.Release|x86.ActiveCfg = Release|Any CPU {F6C974B8-48F8-41C7-95AC-3CFAA720E0E4}.Release|x86.Build.0 = Release|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Debug|x64.ActiveCfg = Debug|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Debug|x64.Build.0 = Debug|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Debug|x86.ActiveCfg = Debug|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Debug|x86.Build.0 = Debug|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Release|Any CPU.Build.0 = Release|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Release|x64.ActiveCfg = Release|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Release|x64.Build.0 = Release|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Release|x86.ActiveCfg = Release|Any CPU + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1308,6 +1322,7 @@ Global {A640E39E-F45C-4AE9-AABF-7F1432D357DA} = {D3AF8295-AEB5-4324-AA82-FCC0014AC310} {B9091AE9-B257-4D3A-A9BC-EE2B43AF57A8} = {A640E39E-F45C-4AE9-AABF-7F1432D357DA} {F6C974B8-48F8-41C7-95AC-3CFAA720E0E4} = {A640E39E-F45C-4AE9-AABF-7F1432D357DA} + {6FA94D22-9369-4A60-BBC1-764CA68F4ED1} = {73108242-625A-4D7B-AA09-63375DBAE464} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {03AD8E21-7507-4E68-A4E9-F4A7E7273164} diff --git a/src/benchmark/Akka.Cluster.Cpu.Benchmark/Akka.Cluster.Cpu.Benchmark.csproj b/src/benchmark/Akka.Cluster.Cpu.Benchmark/Akka.Cluster.Cpu.Benchmark.csproj new file mode 100644 index 00000000000..048352d87ac --- /dev/null +++ b/src/benchmark/Akka.Cluster.Cpu.Benchmark/Akka.Cluster.Cpu.Benchmark.csproj @@ -0,0 +1,18 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + + + + + diff --git a/src/benchmark/Akka.Cluster.Cpu.Benchmark/BenchmarkNode.cs b/src/benchmark/Akka.Cluster.Cpu.Benchmark/BenchmarkNode.cs new file mode 100644 index 00000000000..378af47e3a4 --- /dev/null +++ b/src/benchmark/Akka.Cluster.Cpu.Benchmark/BenchmarkNode.cs @@ -0,0 +1,71 @@ +// //----------------------------------------------------------------------- +// // +// // Copyright (C) 2009-2022 Lightbend Inc. +// // Copyright (C) 2013-2022 .NET Foundation +// // +// //----------------------------------------------------------------------- + +using System; +using System.Linq; +using System.Threading.Tasks; +using Akka.Actor; +using Akka.Configuration; + +namespace Akka.Cluster.Cpu.Benchmark +{ + public class BenchmarkNode + { + private const string Address = "127.0.0.1"; + private const int BasePort = 15225; + + public static async Task EntryPoint(string[] args) + { + var node = new BenchmarkNode(int.Parse(args[1])); + node.Start(); + + // wait forever until we get killed + await Task.Delay(TimeSpan.FromDays(1)); + + return 0; + } + + private readonly Config _config; + private ActorSystem _actorSystem; + + public BenchmarkNode(int nodeOffset) + { + _config = ConfigurationFactory.ParseString($@" +akka {{ + log-dead-letters = off + log-dead-letters-during-shutdown = off + + actor.provider = cluster + + remote {{ + # log-remote-lifecycle-events = DEBUG + dot-netty.tcp {{ + transport-class = ""Akka.Remote.Transport.DotNetty.TcpTransport, Akka.Remote"" + applied-adapters = [] + transport-protocol = tcp + hostname = ""0.0.0.0"" + public-hostname = {Address} + port = {BasePort + nodeOffset} + }} + }} + + cluster {{ + seed-nodes = [""{new Address("akka.tcp", nameof(BenchmarkNode), Address, BasePort)}""] + roles = [benchmark-node] + }} +}}"); + } + + public void Start() + { + _actorSystem = ActorSystem.Create(nameof(BenchmarkNode), _config); + } + + public async Task StopAsync() + => await CoordinatedShutdown.Get(_actorSystem).Run(CoordinatedShutdown.ClrExitReason.Instance); + } +} \ No newline at end of file diff --git a/src/benchmark/Akka.Cluster.Cpu.Benchmark/Program.cs b/src/benchmark/Akka.Cluster.Cpu.Benchmark/Program.cs new file mode 100644 index 00000000000..d36c5f0e32a --- /dev/null +++ b/src/benchmark/Akka.Cluster.Cpu.Benchmark/Program.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NDesk.Options; +using Tmds.Utils; +using Universe.CpuUsage; + +namespace Akka.Cluster.Cpu.Benchmark +{ + public static class Program + { + private const int DefaultSampleDuration = 5; // in seconds + private const int DefaultDelay = 5; // in seconds + private const int DefaultRepeat = 60; + private const int DefaultClusterSize = 9; + + private const int DefaultWarmUpRepeat = 5; + + private static readonly List Usages = new List(); + private static readonly List Processes = new List(); + + public static async Task Main(string[] args) + { + // ExecFunction hook + if (ExecFunction.IsExecFunctionCommand(args)) + return ExecFunction.Program.Main(args); + + // Argument parsing + var sampleDuration = DefaultSampleDuration; + var delay = DefaultDelay; + var repeat = DefaultRepeat; + var clusterSize = DefaultClusterSize; + var warmUp = DefaultWarmUpRepeat; + var showHelp = false; + + var optionSet = new OptionSet() + .Add( + "d|sample-duration=", + "Sets the sample point duration in seconds. Default: 5 seconds", + s => { sampleDuration = int.Parse(s); }) + .Add( + "delay=", + "Sets the initial delay to wait for cluster to stabilize before benchmark starts in seconds. Default: 5 seconds", + s => { delay = int.Parse(s); }) + .Add( + "s|samples=", + "Sets how many samples are taken during the benchmark. Default: 60 samples", + s => { repeat = int.Parse(s); }) + .Add( + "c|cluster-size=", + "Sets how many nodes to be added to the cluster in addition to the tested node. Default: 9 nodes", + s => { clusterSize = int.Parse(s); }) + .Add( + "w|warm-up-count=", + "Sets how many blank samples to be performed as benchmark warm-up before benchmark starts. Default: 5 samples", + s => { warmUp = int.Parse(s); }) + .Add( + "h|?|help", + "Shows help", + s => { showHelp = s != null; }); + + optionSet.Parse(args); + + if (showHelp) + { + Console.WriteLine("Usage: dotnet run -c Release"); + Console.WriteLine("Usage: dotnet run -c Release -- [OPTIONS]"); + Console.WriteLine("Usage: Akka.Cluster.Cpu.Benchmark [OPTIONS]"); + Console.WriteLine("Options:"); + optionSet.WriteOptionDescriptions(Console.Out); + return 0; + } + + // Start the benchmark node + var node = new BenchmarkNode(0); + node.Start(); + + var executor = new FunctionExecutor(o => + { + o.StartInfo.RedirectStandardError = true; + o.OnExit = p => + { + if (p.ExitCode != 0) + { + var message = + "Function execution failed with exit code: " + + $"{p.ExitCode}{Environment.NewLine}{p.StandardError.ReadToEnd()}"; + throw new Exception(message); + } + }; + }); + + // Spin up cluster nodes + foreach (var portOffset in Enumerable.Range(1, clusterSize)) + { + Processes.Add(executor.Start(BenchmarkNode.EntryPoint, new [] + { + portOffset.ToString() + })); + } + + // Wait until things settles down + await Task.Delay(TimeSpan.FromSeconds(delay)); + + // Warm up + foreach (var i in Enumerable.Range(1, warmUp)) + { + var start = CpuUsage.GetByProcess(); + await Task.Delay(TimeSpan.FromSeconds(sampleDuration)); + var end = CpuUsage.GetByProcess(); + var final = end - start; + + Console.WriteLine($"{i}. [Warmup] {final}"); + } + Console.WriteLine(); + + // Start benchmark + foreach (var i in Enumerable.Range(1, repeat)) + { + var start = CpuUsage.GetByProcess(); + await Task.Delay(TimeSpan.FromSeconds(sampleDuration)); + var end = CpuUsage.GetByProcess(); + var final = end - start; + + Console.WriteLine($"{i}. Cpu Usage: {final}"); + Usages.Add(final.Value); + } + + // Kill cluster node processes + foreach (var process in Processes) + { + process.Kill(); + process.Dispose(); + } + + // Stop benchmark node + await node.StopAsync(); + + // Generate csv report + var now = DateTime.Now; + var sb = new StringBuilder(); + sb.AppendLine($"CPU Benchmark {now}"); + sb.AppendLine($"Sample Time,{sampleDuration},second(s)"); + sb.AppendLine($"Sample points,{repeat}"); + sb.AppendLine($"Cluster size,{clusterSize},node(s)"); + sb.AppendLine("Sample time,User usage,User percent,Kernel usage,Kernel percent,Total usage,Total percent"); + foreach (var iter in Enumerable.Range(1, repeat)) + { + var usage = Usages[iter - 1]; + var user = usage.UserUsage.TotalSeconds; + var kernel = usage.KernelUsage.TotalSeconds; + var total = usage.TotalMicroSeconds / 1000000.0; + sb.AppendLine($"{iter * sampleDuration},{user},{(user/sampleDuration)*100},{kernel},{(kernel/sampleDuration)*100},{total},{(total/sampleDuration)*100}"); + } + + await File.WriteAllTextAsync($"CpuBenchmark_{now.ToFileTime()}.csv", sb.ToString()); + + // Generate console report + sb.Clear(); + sb.AppendLine("CPU Benchmark complete."); + sb.AppendLine(); + + sb.AppendLine() + .AppendLine(" CPU Usage Mode | Mean | StdErr | StdDev | Median | Maximum |") + .AppendLine("--------------- |----- |------- |------- |------- |-------- |") + .AppendLine(CalculateResult(Usages.Select(u => u.UserUsage.TotalMicroSeconds), "User")) + .AppendLine(CalculateResult(Usages.Select(u => u.KernelUsage.TotalMicroSeconds), "Kernel")) + .AppendLine(CalculateResult(Usages.Select(u => u.TotalMicroSeconds), "Total")); + + Console.WriteLine(sb.ToString()); + + return 0; + } + + private static string CalculateResult(IEnumerable values, string name) + { + var times = values.OrderBy(i => i).ToArray(); + var medianIndex = times.Length / 2; + + var mean = times.Average(); + var stdDev = Math.Sqrt(times.Average(v => Math.Pow(v - mean, 2))); + var stdErr = stdDev / Math.Sqrt(times.Length); + double median; + if (times.Length % 2 == 0) + median = (times[medianIndex - 1] + times[medianIndex]) / 2.0; + else + median = times[medianIndex]; + + return $" {name} | {(mean / 1000.0):N3} ms | {(stdErr / 1000.0):N3} ms | {(stdDev / 1000.0):N3} ms | {(median / 1000.0):N3} ms | {(times.Last() / 1000.0):N3} ms |"; + } + } +} + diff --git a/src/benchmark/Akka.Cluster.Cpu.Benchmark/README.md b/src/benchmark/Akka.Cluster.Cpu.Benchmark/README.md new file mode 100644 index 00000000000..49c29dd3c37 --- /dev/null +++ b/src/benchmark/Akka.Cluster.Cpu.Benchmark/README.md @@ -0,0 +1,36 @@ +# Akka.Cluster.Cpu.Benchmark + +This project is a standalone console CPU benchmark to measure Akka.Cluster actors CPU usage. It will: + +* Spin up a single bare minimum cluster node as a seed node, +* Spins up a cluster with a predetermined size that uses that node as seed, and then +* Collect CPU usage at regular interval and save the result in a comma separated value (.csv) file that can be imported into a spreadsheet application. + +## Usage + +To run this project directly using .NET CLI, use one of the following commands: + +```powershell +dotnet run -c Release +dotnet run -c Release -- [OPTIONS] +``` + +To run this as a standalone compiled executable, use one of the following commands: + +```powershell +./Akka.Cluster.Cpu.Benchmark.exe [OPTIONS] +dotnet run Akka.Cluster.Cpu.Benchmark.dll +dotnet run Akka.Cluster.Cpu.Benchmark.dll -- [OPTIONS] +``` + +## Options + +| Option | Description | +|-----------------------------|--------------------------------------------------------------------------------------------------------------------| +| -d, --sample-duration=VALUE | Sets the sample point duration in seconds.
Default: 5 seconds | +| --delay=VALUE | Sets the initial delay to wait for cluster to stabilize before benchmark starts in seconds.
Default: 5 seconds | +| -s, --samples=VALUE | Sets how many samples are taken during the benchmark.
Default: 60 samples | +| -c, --cluster-size=VALUE | Sets how many nodes to be added to the cluster in addition to the tested node.
Default: 9 nodes | +| -w, --warm-up-count=VALUE | Sets how many blank samples to be performed as benchmark warm-up before benchmark starts.
Default: 5 samples | +| -h, -?, --help | Shows help | +