Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Benchmarking improvements #7971

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 25 additions & 16 deletions src/Nethermind/Nethermind.Benchmark.Runner/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
using BenchmarkDotNet.Running;
using System.Linq;
using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
using BenchmarkDotNet.Columns;
using Nethermind.Precompiles.Benchmark;

namespace Nethermind.Benchmark.Runner
{
Expand All @@ -23,34 +25,39 @@ public DashboardConfig(params Job[] jobs)
AddJob(job.WithToolchain(InProcessNoEmitToolchain.Instance));
}

AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Descriptor);
AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Statistics);
AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Params);
AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Metrics);
AddColumnProvider(DefaultColumnProviders.Descriptor);
AddColumnProvider(DefaultColumnProviders.Statistics);
AddColumnProvider(DefaultColumnProviders.Params);
AddColumnProvider(DefaultColumnProviders.Metrics);
AddLogger(BenchmarkDotNet.Loggers.ConsoleLogger.Default);
AddExporter(BenchmarkDotNet.Exporters.Json.JsonExporter.FullCompressed);
AddDiagnoser(BenchmarkDotNet.Diagnosers.MemoryDiagnoser.Default);
WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(100));
}
}

public class PrecompileBenchmarkConfig : DashboardConfig
{
public PrecompileBenchmarkConfig() : base(Job.MediumRun.WithRuntime(CoreRuntime.Core90))
{
AddColumnProvider(new GasColumnProvider());
}
}

public static class Program
{
public static void Main(string[] args)
{
List<Assembly> additionalJobAssemblies = new()
{
typeof(Nethermind.JsonRpc.Benchmark.EthModuleBenchmarks).Assembly,
typeof(Nethermind.Benchmarks.Core.Keccak256Benchmarks).Assembly,
typeof(Nethermind.Evm.Benchmark.EvmStackBenchmarks).Assembly,
typeof(Nethermind.Network.Benchmarks.DiscoveryBenchmarks).Assembly,
typeof(Nethermind.Precompiles.Benchmark.KeccakBenchmark).Assembly
};
List<Assembly> additionalJobAssemblies = [
typeof(JsonRpc.Benchmark.EthModuleBenchmarks).Assembly,
typeof(Benchmarks.Core.Keccak256Benchmarks).Assembly,
typeof(Evm.Benchmark.EvmStackBenchmarks).Assembly,
typeof(Network.Benchmarks.DiscoveryBenchmarks).Assembly,
];

List<Assembly> simpleJobAssemblies = new()
{
typeof(Nethermind.EthereumTests.Benchmark.EthereumTests).Assembly,
};
List<Assembly> simpleJobAssemblies = [
// typeof(EthereumTests.Benchmark.EthereumTests).Assembly,
];

if (Debugger.IsAttached)
{
Expand All @@ -67,6 +74,8 @@ public static void Main(string[] args)
{
BenchmarkRunner.Run(assembly, new DashboardConfig(), args);
}

BenchmarkRunner.Run(typeof(KeccakBenchmark).Assembly, new PrecompileBenchmarkConfig(), args);
}
}
}
Expand Down
116 changes: 116 additions & 0 deletions src/Nethermind/Nethermind.Precompiles.Benchmark/GasColumnProvider.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

using System.Collections.Generic;
using System.Linq;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
using Nethermind.Specs.Forks;
using BenchmarkDotNet.Mathematics;
using Perfolizer.Mathematics.Common;

namespace Nethermind.Precompiles.Benchmark;

public class GasColumnProvider : IColumnProvider
{
private static readonly IColumn[] Columns = [
new GasColumn(),
new GasThroughputColumn(),
new GasConfidenceIntervalColumn(true), // Lower bound
new GasConfidenceIntervalColumn(false) // Upper bound
];

public IEnumerable<IColumn> GetColumns(Summary summary) => Columns;

private abstract class BaseGasColumn : IColumn
{
public bool AlwaysShow => true;
public ColumnCategory Category => ColumnCategory.Custom;
public int PriorityInCategory => 0;
public bool IsNumeric => true;
public UnitType UnitType => UnitType.Size;

public abstract string Id { get; }
public abstract string ColumnName { get; }
public abstract string Legend { get; }

protected static (long? gas, Statistics? stats) GetBenchmarkData(Summary summary, BenchmarkCase benchmarkCase)
{
BenchmarkDotNet.Parameters.ParameterInstance? inputParam = benchmarkCase.Parameters.Items.FirstOrDefault(p => p.Name == "Input");
var gas = ((PrecompileBenchmarkBase.Param)inputParam!.Value).Gas(Cancun.Instance);
Statistics? stats = summary.Reports.FirstOrDefault(r => r.BenchmarkCase == benchmarkCase)?.ResultStatistics;
return (gas, stats);
}

protected static double CalculateMGasThroughput(long gas, double nanoseconds)
{
double opThroughput = 1_000_000_000.0 / nanoseconds;
return gas * opThroughput / 1_000_000.0;
}

public abstract string GetValue(Summary summary, BenchmarkCase benchmarkCase);

public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
=> GetValue(summary, benchmarkCase);

public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false;

public bool IsAvailable(Summary summary) => true;
}

private class GasColumn : BaseGasColumn
{
public override string Id => "Gas";
public override string ColumnName => "Gas";
public override string Legend => "Amount of gas used by the operation";

public override string GetValue(Summary summary, BenchmarkCase benchmarkCase)
{
(long? gas, Statistics? _) = GetBenchmarkData(summary, benchmarkCase);
return gas?.ToString() ?? "N/A";
}
}

private class GasThroughputColumn : BaseGasColumn
{
public override string Id => "GasThroughput";
public override string ColumnName => "Throughput";
public override string Legend => "Amount of gas processed per second";

public override string GetValue(Summary summary, BenchmarkCase benchmarkCase)
{
(long? gas, Statistics? stats) = GetBenchmarkData(summary, benchmarkCase);

if (gas is null || stats?.Mean is null)
{
return "N/A";
}

double mgasThroughput = CalculateMGasThroughput(gas.Value, stats.Mean);
return mgasThroughput.ToString("F2") + " MGas/s";
}
}

private class GasConfidenceIntervalColumn(bool isLower) : BaseGasColumn
{
public override string Id => isLower ? "GasCI-Lower" : "GasCI-Upper";
public override string ColumnName => isLower ? "Throughput CI-Lower" : "Throughput CI-Upper";
public override string Legend => $"{(isLower ? "Lower" : "Upper")} bound of gas throughput 99% confidence interval";

public override string GetValue(Summary summary, BenchmarkCase benchmarkCase)
{
(long? gas, Statistics? stats) = GetBenchmarkData(summary, benchmarkCase);

if (gas is null || stats?.Mean is null)
{
return "N/A";
}

ConfidenceInterval ci = stats.GetConfidenceInterval(ConfidenceLevel.L99);
double bound = isLower ? ci.Lower : ci.Upper;
double mgasThroughput = CalculateMGasThroughput(gas.Value, bound);
return mgasThroughput.ToString("F2") + " MGas/s";
}
}
}
5 changes: 3 additions & 2 deletions src/Nethermind/Nethermind.Precompiles.Benchmark/JsonInput.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// ReSharper disable UnusedAutoPropertyAccessor.Global
// ReSharper disable UnusedMember.Global
// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

namespace Nethermind.Precompiles.Benchmark
{
public class JsonInput
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public class KeccakBenchmark
{
public readonly struct Param
{
private static Random _random = new Random(42);
private static readonly Random _random = new(42);

public Param(byte[] bytes)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,48 +3,37 @@

using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using BenchmarkDotNet.Attributes;
using Nethermind.Core.Extensions;
using Nethermind.Core.Specs;
using Nethermind.Evm.Precompiles;
using Nethermind.Serialization.Json;
using Nethermind.Specs.Forks;

namespace Nethermind.Precompiles.Benchmark
{
[SuppressMessage("ReSharper", "UnassignedGetOnlyAutoProperty")]
[SuppressMessage("ReSharper", "MemberCanBePrivate.Global")]
[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")]
public abstract class PrecompileBenchmarkBase
{
protected abstract IEnumerable<IPrecompile> Precompiles { get; }

protected abstract string InputsDirectory { get; }

public readonly struct Param
public readonly struct Param(IPrecompile precompile, string name, byte[] bytes, byte[]? expected)
{
public IPrecompile Precompile { get; }
public IPrecompile Precompile { get; } = precompile ?? throw new ArgumentNullException(nameof(precompile));

public Param(IPrecompile precompile, string name, byte[] bytes, byte[]? expected)
{
Precompile = precompile ?? throw new ArgumentNullException(nameof(precompile));
Bytes = bytes;
Name = name;
ExpectedResult = expected;
}
public byte[] Bytes { get; } = bytes;

public byte[] Bytes { get; }
public byte[]? ExpectedResult { get; } = expected;

public byte[]? ExpectedResult { get; }
public string Name { get; } = name;

public string Name { get; }
public long Gas(IReleaseSpec releaseSpec) =>
precompile.BaseGasCost(releaseSpec) + precompile.DataGasCost(Bytes, releaseSpec);

public override string ToString()
{
return Name;
}
public override string ToString() => Name;
}

public IEnumerable<Param> Inputs
Expand All @@ -53,7 +42,7 @@ public IEnumerable<Param> Inputs
{
foreach (IPrecompile precompile in Precompiles)
{
List<Param> inputs = new List<Param>();
List<Param> inputs = [];
foreach (string file in Directory.GetFiles($"{InputsDirectory}/current", "*.csv", SearchOption.TopDirectoryOnly))
{
// take only first line from each file
Expand All @@ -64,9 +53,9 @@ public IEnumerable<Param> Inputs

foreach (string file in Directory.GetFiles($"{InputsDirectory}/current", "*.json", SearchOption.TopDirectoryOnly))
{
EthereumJsonSerializer jsonSerializer = new EthereumJsonSerializer();
var jsonInputs = jsonSerializer.Deserialize<JsonInput[]>(File.ReadAllText(file));
var parameters = jsonInputs.Select(i => new Param(precompile, i.Name!, i.Input!, i.Expected));
EthereumJsonSerializer jsonSerializer = new();
JsonInput[] jsonInputs = jsonSerializer.Deserialize<JsonInput[]>(File.ReadAllText(file));
IEnumerable<Param> parameters = jsonInputs.Select(i => new Param(precompile, i.Name!, i.Input!, i.Expected));
inputs.AddRange(parameters);
}

Expand All @@ -82,14 +71,10 @@ public IEnumerable<Param> Inputs
public Param Input { get; set; }

private static byte[] LineToTestInput(string line)
{
return Bytes.FromHexString(line.Split(',')[0]);
}
=> Bytes.FromHexString(line.Split(',')[0]);

[Benchmark(Baseline = true)]
public (ReadOnlyMemory<byte>, bool) Baseline()
{
return Input.Precompile.Run(Input.Bytes, Berlin.Instance);
}
=> Input.Precompile.Run(Input.Bytes, Cancun.Instance);
}
}
Loading