From 922dfff62d6cf6fd808865e705a09eee63690a2e Mon Sep 17 00:00:00 2001 From: Evgeniy Andreev Date: Sun, 14 Oct 2018 20:02:19 +0400 Subject: [PATCH] Added [ParamsAllValues] (#908), fixes #658 * Added [ParamsAllValues] * #658: restore lost ParamsAllValues attribute * #658: fix sample * #658: remove attribute's duplicate * #658: add tests and validator for ParamsAllValues * #658: update docs * #658: remove integration tests * #658: refactor ParamsAllValues validator * #658: update docs --- docs/articles/features/parameterization.md | 2 + docs/articles/samples/IntroParamsAllValues.md | 42 +++++ .../IntroParamsAllValues.cs | 28 ++++ .../Attributes/ParamsAllValuesAttribute.cs | 8 + .../Validators/ParamsAllValuesValidator.cs | 64 ++++++++ src/BenchmarkDotNet/Configs/DefaultConfig.cs | 1 + .../Running/BenchmarkConverter.cs | 60 ++++--- ...ceSummary.WithAllValuesOfBool.approved.txt | 14 ++ ...ceSummary.WithAllValuesOfEnum.approved.txt | 15 ++ ...y.WithAllValuesOfNullableBool.approved.txt | 15 ++ ...y.WithAllValuesOfNullableEnum.approved.txt | 16 ++ ....WithNotAllowedFlagsEnumError.approved.txt | 14 ++ ...thNotAllowedNullableTypeError.approved.txt | 15 ++ ...mmary.WithNotAllowedTypeError.approved.txt | 14 ++ .../ParamsAllValuesApprovalTests.cs | 148 ++++++++++++++++++ .../BenchmarkDotNet.Tests.csproj | 3 + 16 files changed, 439 insertions(+), 20 deletions(-) create mode 100644 docs/articles/samples/IntroParamsAllValues.md create mode 100644 samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs create mode 100644 src/BenchmarkDotNet/Attributes/ParamsAllValuesAttribute.cs create mode 100644 src/BenchmarkDotNet/Attributes/Validators/ParamsAllValuesValidator.cs create mode 100644 tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfBool.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfEnum.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableBool.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableEnum.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedFlagsEnumError.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedNullableTypeError.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedTypeError.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesApprovalTests.cs diff --git a/docs/articles/features/parameterization.md b/docs/articles/features/parameterization.md index 0e1c3c7690..82d0a8c40a 100644 --- a/docs/articles/features/parameterization.md +++ b/docs/articles/features/parameterization.md @@ -11,6 +11,8 @@ name: Benchmark Parameterization [!include[IntroParamsSource](../samples/IntroParamsSource.md)] +[!include[IntroParamsAllValues](../samples/IntroParamsAllValues.md)] + [!include[IntroArguments](../samples/IntroArguments.md)] [!include[IntroArgumentsSource](../samples/IntroArgumentsSource.md)] diff --git a/docs/articles/samples/IntroParamsAllValues.md b/docs/articles/samples/IntroParamsAllValues.md new file mode 100644 index 0000000000..737e7bd6b3 --- /dev/null +++ b/docs/articles/samples/IntroParamsAllValues.md @@ -0,0 +1,42 @@ +--- +uid: BenchmarkDotNet.Samples.IntroParamsAllValues +--- + +## Sample: IntroParamsAllValues + +If you want to use all possible values of an `enum` or another type with a small number of values, you can use the [`[ParamsAllValues]`](xref:BenchmarkDotNet.Attributes.ParamsAllValuesAttribute) attribute, instead of listing all the values by hand. The types supported by the attribute are: + +* `bool` +* any `enum` that is not marked with `[Flags]` +* `Nullable`, where `T` is an enum or boolean + +### Source code + +[!code-csharp[IntroParamsAllValues.cs](../../../samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs)] + +### Output + +```markdown + Method | E | B | Mean | Error | +---------- |---- |------ |---------:|------:| + Benchmark | A | ? | 101.9 ms | NA | + Benchmark | A | False | 111.9 ms | NA | + Benchmark | A | True | 122.3 ms | NA | + Benchmark | BB | ? | 201.5 ms | NA | + Benchmark | BB | False | 211.8 ms | NA | + Benchmark | BB | True | 221.4 ms | NA | + Benchmark | CCC | ? | 301.8 ms | NA | + Benchmark | CCC | False | 312.3 ms | NA | + Benchmark | CCC | True | 322.2 ms | NA | + +// * Legends * + E : Value of the 'E' parameter + B : Value of the 'B' parameter +``` + +### Links + +* @docs.parameterization +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroParamsAllValues + +--- \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs b/samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs new file mode 100644 index 0000000000..d30437cc47 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs @@ -0,0 +1,28 @@ +using System.Threading; +using BenchmarkDotNet.Attributes; + +namespace BenchmarkDotNet.Samples +{ + [DryJob] + public class IntroParamsAllValues + { + public enum CustomEnum + { + A, + BB, + CCC + } + + [ParamsAllValues] + public CustomEnum E { get; set; } + + [ParamsAllValues] + public bool? B { get; set; } + + [Benchmark] + public void Benchmark() + { + Thread.Sleep(E.ToString().Length * 100 + (B == true ? 20 : B == false ? 10 : 0)); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/ParamsAllValuesAttribute.cs b/src/BenchmarkDotNet/Attributes/ParamsAllValuesAttribute.cs new file mode 100644 index 0000000000..2724496626 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/ParamsAllValuesAttribute.cs @@ -0,0 +1,8 @@ +using System; +namespace BenchmarkDotNet.Attributes +{ + [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)] + public class ParamsAllValuesAttribute : Attribute + { + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/Validators/ParamsAllValuesValidator.cs b/src/BenchmarkDotNet/Attributes/Validators/ParamsAllValuesValidator.cs new file mode 100644 index 0000000000..6f84d3a0d7 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Validators/ParamsAllValuesValidator.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Extensions; + +namespace BenchmarkDotNet.Validators +{ + class ParamsAllValuesValidator : IValidator + { + public static readonly ParamsAllValuesValidator FailOnError = new ParamsAllValuesValidator(); + + public bool TreatsWarningsAsErrors => true; + + private ParamsAllValuesValidator() { } + + private const BindingFlags reflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; + + public IEnumerable Validate(ValidationParameters input) => + input.Benchmarks + .Select(benchmark => benchmark.Descriptor.Type) + .Distinct() + .SelectMany(type => type.GetTypeMembersWithGivenAttribute(reflectionFlags)) + .Distinct() + .Select(member => GetErrorOrDefault(member.ParameterType)) + .Where(error => error != default); + + private bool IsBool(Type paramType) => paramType == typeof(bool); + private bool IsEnum(Type paramType) => paramType.GetTypeInfo().IsEnum; + private bool IsFlagsEnum(Type paramType) + { + var typeInfo = paramType.GetTypeInfo(); + return typeInfo.IsEnum && typeInfo.IsDefined(typeof(FlagsAttribute)); + } + private bool IsNullable(Type paramType, out Type underlyingType) + { + underlyingType = Nullable.GetUnderlyingType(paramType); + return underlyingType != null; + } + + private ValidationError GetErrorOrDefault(Type parameterType) + { + switch (parameterType) + { + case Type t when IsNullable(t, out Type underType): + return GetErrorOrDefault(underType); + + case Type t when IsFlagsEnum(t): + return new ValidationError( + TreatsWarningsAsErrors, + $"Unable to use {parameterType.Name} with [ParamsAllValues], because it's flags enum."); + + case Type t when IsBool(t) || IsEnum(t): + return default; + + default: + return new ValidationError( + TreatsWarningsAsErrors, + $"Type {parameterType.Name} cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type."); + } + } + } +} diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index a44fafcba9..ae922baec3 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -62,6 +62,7 @@ public IEnumerable GetValidators() yield return RunModeValidator.FailOnError; yield return GenericBenchmarksValidator.DontFailOnError; yield return DeferredExecutionValidator.FailOnError; + yield return ParamsAllValuesValidator.FailOnError; } public IEnumerable GetJobs() => Array.Empty(); diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs index 0846a6ed45..8889536866 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs @@ -161,30 +161,30 @@ private static Descriptor CreateDescriptor( private static ParameterDefinitions GetParameterDefinitions(Type type) { - const BindingFlags reflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; - - var allParamsMembers = type.GetTypeMembersWithGivenAttribute(reflectionFlags); - - var allParamsSourceMembers = type.GetTypeMembersWithGivenAttribute(reflectionFlags); + IEnumerable GetDefinitions(Func getValidValues) where TAttribute : Attribute + { + const BindingFlags reflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; - var definitions = allParamsMembers - .Select(member => + var allMembers = type.GetTypeMembersWithGivenAttribute(reflectionFlags); + return allMembers.Select(member => new ParameterDefinition( member.Name, member.IsStatic, - GetValidValues(member.Attribute.Values, member.ParameterType), - false)) - .Concat(allParamsSourceMembers.Select(member => - { - var paramsValues = GetValidValuesForParamsSource(type, member.Attribute.Name); - return new ParameterDefinition( - member.Name, - member.IsStatic, - SmartParamBuilder.CreateForParams(paramsValues.source, paramsValues.values), - false); - - })) - .ToArray(); + getValidValues(member.Attribute, member.ParameterType), + false)); + } + + var paramsDefinitions = GetDefinitions((attribute, parameterType) => GetValidValues(attribute.Values, parameterType)); + + var paramsSourceDefinitions = GetDefinitions((attribute, _) => + { + var paramsValues = GetValidValuesForParamsSource(type, attribute.Name); + return SmartParamBuilder.CreateForParams(paramsValues.source, paramsValues.values); + }); + + var paramsAllValuesDefinitions = GetDefinitions((_, paramaterType) => GetAllValidValues(paramaterType)); + + var definitions = paramsDefinitions.Concat(paramsSourceDefinitions).Concat(paramsAllValuesDefinitions).ToArray(); return new ParameterDefinitions(definitions); } @@ -314,5 +314,25 @@ private static object[] ToArray(object sourceValue, MemberInfo memberInfo, Type return collection.Cast().ToArray(); } + + private static object[] GetAllValidValues(Type parameterType) + { + if (parameterType == typeof(bool)) + return new object[] { false, true }; + + if (parameterType.GetTypeInfo().IsEnum) + { + if (parameterType.GetTypeInfo().IsDefined(typeof(FlagsAttribute))) + return new object[] { Activator.CreateInstance(parameterType) }; + + return Enum.GetValues(parameterType).Cast().ToArray(); + } + + var nullableUnderlyingType = Nullable.GetUnderlyingType(parameterType); + if (nullableUnderlyingType != null) + return new object[] { null }.Concat(GetAllValidValues(nullableUnderlyingType)).ToArray(); + + return new object[] { Activator.CreateInstance(parameterType) }; + } } } diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfBool.approved.txt b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfBool.approved.txt new file mode 100644 index 0000000000..60b1e1d906 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfBool.approved.txt @@ -0,0 +1,14 @@ +=== WithAllValuesOfBool === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + + + Method | ParamProperty | Mean | Error | StdDev | +---------- |-------------- |---------:|---------:|---------:| + Benchmark | False | 102.0 ns | 6.088 ns | 1.581 ns | ^ + Benchmark | True | 202.0 ns | 6.088 ns | 1.581 ns | ^ + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfEnum.approved.txt b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfEnum.approved.txt new file mode 100644 index 0000000000..eea5d7eec4 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfEnum.approved.txt @@ -0,0 +1,15 @@ +=== WithAllValuesOfEnum === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + + + Method | ParamProperty | Mean | Error | StdDev | +---------- |-------------- |---------:|---------:|---------:| + Benchmark | A | 102.0 ns | 6.088 ns | 1.581 ns | ^ + Benchmark | B | 202.0 ns | 6.088 ns | 1.581 ns | ^ + Benchmark | C | 302.0 ns | 6.088 ns | 1.581 ns | ^ + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableBool.approved.txt b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableBool.approved.txt new file mode 100644 index 0000000000..efc36b851d --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableBool.approved.txt @@ -0,0 +1,15 @@ +=== WithAllValuesOfNullableBool === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + + + Method | ParamProperty | Mean | Error | StdDev | +---------- |-------------- |---------:|---------:|---------:| + Benchmark | ? | 102.0 ns | 6.088 ns | 1.581 ns | ^ + Benchmark | False | 202.0 ns | 6.088 ns | 1.581 ns | ^ + Benchmark | True | 302.0 ns | 6.088 ns | 1.581 ns | ^ + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableEnum.approved.txt b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableEnum.approved.txt new file mode 100644 index 0000000000..27fbd7a6de --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableEnum.approved.txt @@ -0,0 +1,16 @@ +=== WithAllValuesOfNullableEnum === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + + + Method | ParamProperty | Mean | Error | StdDev | +---------- |-------------- |---------:|---------:|---------:| + Benchmark | ? | 102.0 ns | 6.088 ns | 1.581 ns | ^ + Benchmark | A | 202.0 ns | 6.088 ns | 1.581 ns | ^ + Benchmark | B | 302.0 ns | 6.088 ns | 1.581 ns | ^ + Benchmark | C | 402.0 ns | 6.088 ns | 1.581 ns | ^ + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedFlagsEnumError.approved.txt b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedFlagsEnumError.approved.txt new file mode 100644 index 0000000000..a7f937ea68 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedFlagsEnumError.approved.txt @@ -0,0 +1,14 @@ +=== WithNotAllowedFlagsEnumError === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + + + Method | ParamProperty | Mean | Error | StdDev | +---------- |-------------- |---------:|---------:|---------:| + Benchmark | 0 | 102.0 ns | 6.088 ns | 1.581 ns | + +Errors: 1 +* Unable to use TestFlagsEnum with [ParamsAllValues], because it's flags enum. diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedNullableTypeError.approved.txt b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedNullableTypeError.approved.txt new file mode 100644 index 0000000000..4a0ba5ea55 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedNullableTypeError.approved.txt @@ -0,0 +1,15 @@ +=== WithNotAllowedNullableTypeError === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + + + Method | ParamProperty | Mean | Error | StdDev | +---------- |-------------- |---------:|---------:|---------:| + Benchmark | ? | 102.0 ns | 6.088 ns | 1.581 ns | ^ + Benchmark | 0 | 202.0 ns | 6.088 ns | 1.581 ns | ^ + +Errors: 1 +* Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type. diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedTypeError.approved.txt b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedTypeError.approved.txt new file mode 100644 index 0000000000..0e4746173a --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedTypeError.approved.txt @@ -0,0 +1,14 @@ +=== WithNotAllowedTypeError === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + + + Method | ParamProperty | Mean | Error | StdDev | +---------- |-------------- |---------:|---------:|---------:| + Benchmark | 0 | 102.0 ns | 6.088 ns | 1.581 ns | + +Errors: 1 +* Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type. diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesApprovalTests.cs b/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesApprovalTests.cs new file mode 100644 index 0000000000..9b5b10dad3 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesApprovalTests.cs @@ -0,0 +1,148 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using ApprovalTests; +using ApprovalTests.Namers; +using ApprovalTests.Reporters; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Validators; +using JetBrains.Annotations; +using Xunit; +using System.Collections.Generic; + +namespace BenchmarkDotNet.Tests.Attributes +{ + // In case of failed approval tests, use the following reporter: + // [UseReporter(typeof(KDiffReporter))] + [UseReporter(typeof(XUnit2Reporter))] + [UseApprovalSubdirectory("ApprovedFiles")] + [Collection("ApprovalTests")] + public class ParamsAllValuesApprovalTests : IDisposable + { + private readonly CultureInfo initCulture; + + public ParamsAllValuesApprovalTests() => initCulture = Thread.CurrentThread.CurrentCulture; + + [UsedImplicitly] + public static TheoryData GetBenchmarkTypes() + { + var data = new TheoryData(); + foreach (var type in typeof(Benchmarks).GetNestedTypes()) + data.Add(type); + return data; + } + + [Theory] + [MemberData(nameof(GetBenchmarkTypes))] + [MethodImpl(MethodImplOptions.NoInlining)] + public void BenchmarkShouldProduceSummary(Type benchmarkType) + { + NamerFactory.AdditionalInformation = benchmarkType.Name; + Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = MarkdownExporter.Mock; + var summary = MockFactory.CreateSummary(benchmarkType); + exporter.ExportToLog(summary, logger); + + var validator = ParamsAllValuesValidator.FailOnError; + var errors = validator.Validate(new ValidationParameters(summary.BenchmarksCases, summary.Config)).ToList(); + logger.WriteLine(); + logger.WriteLine("Errors: " + errors.Count); + foreach (var error in errors) + logger.WriteLineError("* " + error.Message); + + Approvals.Verify(logger.GetLog()); + } + + public void Dispose() => Thread.CurrentThread.CurrentCulture = initCulture; + + public enum TestEnum + { + A = 1, B, C + } + + [Flags] + public enum TestFlagsEnum + { + A = 0b001, + B = 0b010, + C = 0b100 + } + + [SuppressMessage("ReSharper", "InconsistentNaming")] + public static class Benchmarks + { + public class WithAllValuesOfBool + { + [ParamsAllValues] + public bool ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() { } + } + + public class WithAllValuesOfEnum + { + [ParamsAllValues] + public TestEnum ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() { } + } + + public class WithAllValuesOfNullableBool + { + [ParamsAllValues] + public bool? ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() { } + } + + public class WithAllValuesOfNullableEnum + { + [ParamsAllValues] + public TestEnum? ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() { } + } + + public class WithNotAllowedTypeError + { + [ParamsAllValues] + public int ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() { } + } + + public class WithNotAllowedNullableTypeError + { + [ParamsAllValues] + public int? ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() { } + } + + public class WithNotAllowedFlagsEnumError + { + [ParamsAllValues] + public TestFlagsEnum ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() { } + } + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj index 813b8f503e..b13ded1600 100644 --- a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj +++ b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj @@ -40,4 +40,7 @@ + + + \ No newline at end of file