Added [ParamsAllValues] (#908), fixes #658

* Added [ParamsAllValues] * #658: restore lost ParamsAllValues attribute * #658: fix sample * #658: remove attribute's duplicate * #658: add tests and validator for ParamsAllValues * #658: update docs * #658: remove integration tests * #658: refactor ParamsAllValues validator * #658: update docs
dotnet · Oct 14, 2018 · 922dfff · 922dfff
1 parent c3b6095
commit 922dfff
Show file tree

Hide file tree

Showing 16 changed files with 439 additions and 20 deletions.
diff --git a/docs/articles/features/parameterization.md b/docs/articles/features/parameterization.md
@@ -11,6 +11,8 @@ name: Benchmark Parameterization
 
 [!include[IntroParamsSource](../samples/IntroParamsSource.md)]
 
+[!include[IntroParamsAllValues](../samples/IntroParamsAllValues.md)]
+
 [!include[IntroArguments](../samples/IntroArguments.md)]
 
 [!include[IntroArgumentsSource](../samples/IntroArgumentsSource.md)]

diff --git a/docs/articles/samples/IntroParamsAllValues.md b/docs/articles/samples/IntroParamsAllValues.md
@@ -0,0 +1,42 @@
+---
+uid: BenchmarkDotNet.Samples.IntroParamsAllValues
+---
+
+## Sample: IntroParamsAllValues
+
+If you want to use all possible values of an `enum` or another type with a small number of values, you can use the [`[ParamsAllValues]`](xref:BenchmarkDotNet.Attributes.ParamsAllValuesAttribute) attribute, instead of listing all the values by hand. The types supported by the attribute are:
+
+* `bool`
+* any `enum` that is not marked with `[Flags]`
+* `Nullable<T>`, where `T` is an enum or boolean
+
+### Source code
+
+[!code-csharp[IntroParamsAllValues.cs](../../../samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs)]
+
+### Output
+
+```markdown
+    Method |   E |     B |     Mean | Error |
+---------- |---- |------ |---------:|------:|
+ Benchmark |   A |     ? | 101.9 ms |    NA |
+ Benchmark |   A | False | 111.9 ms |    NA |
+ Benchmark |   A |  True | 122.3 ms |    NA |
+ Benchmark |  BB |     ? | 201.5 ms |    NA |
+ Benchmark |  BB | False | 211.8 ms |    NA |
+ Benchmark |  BB |  True | 221.4 ms |    NA |
+ Benchmark | CCC |     ? | 301.8 ms |    NA |
+ Benchmark | CCC | False | 312.3 ms |    NA |
+ Benchmark | CCC |  True | 322.2 ms |    NA |
+
+// * Legends *
+  E     : Value of the 'E' parameter
+  B     : Value of the 'B' parameter
+```
+
+### Links
+
+* @docs.parameterization
+* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroParamsAllValues
+
+---
diff --git a/samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs b/samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs
@@ -0,0 +1,28 @@
+using System.Threading;
+using BenchmarkDotNet.Attributes;
+
+namespace BenchmarkDotNet.Samples
+{
+    [DryJob]
+    public class IntroParamsAllValues
+    {
+        public enum CustomEnum
+        {
+            A,
+            BB,
+            CCC
+        }
+
+        [ParamsAllValues]
+        public CustomEnum E { get; set; }
+
+        [ParamsAllValues]
+        public bool? B { get; set; }
+
+        [Benchmark]
+        public void Benchmark()
+        {
+            Thread.Sleep(E.ToString().Length * 100 + (B == true ? 20 : B == false ? 10 : 0));
+        }
+    }
+}
diff --git a/src/BenchmarkDotNet/Attributes/ParamsAllValuesAttribute.cs b/src/BenchmarkDotNet/Attributes/ParamsAllValuesAttribute.cs
@@ -0,0 +1,8 @@
+using System;
+namespace BenchmarkDotNet.Attributes
+{
+    [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)]
+    public class ParamsAllValuesAttribute : Attribute
+    {
+    }
+}
diff --git a/src/BenchmarkDotNet/Attributes/Validators/ParamsAllValuesValidator.cs b/src/BenchmarkDotNet/Attributes/Validators/ParamsAllValuesValidator.cs
@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Extensions;
+
+namespace BenchmarkDotNet.Validators
+{
+    class ParamsAllValuesValidator : IValidator
+    {
+        public static readonly ParamsAllValuesValidator FailOnError = new ParamsAllValuesValidator();
+
+        public bool TreatsWarningsAsErrors => true;
+
+        private ParamsAllValuesValidator() { }
+
+        private const BindingFlags reflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
+
+        public IEnumerable<ValidationError> Validate(ValidationParameters input) => 
+            input.Benchmarks
+                .Select(benchmark => benchmark.Descriptor.Type)
+                .Distinct()
+                .SelectMany(type => type.GetTypeMembersWithGivenAttribute<ParamsAllValuesAttribute>(reflectionFlags))
+                .Distinct()
+                .Select(member => GetErrorOrDefault(member.ParameterType))
+                .Where(error => error != default);
+
+        private bool IsBool(Type paramType) => paramType == typeof(bool);
+        private bool IsEnum(Type paramType) => paramType.GetTypeInfo().IsEnum;
+        private bool IsFlagsEnum(Type paramType)
+        {
+            var typeInfo = paramType.GetTypeInfo();
+            return typeInfo.IsEnum && typeInfo.IsDefined(typeof(FlagsAttribute));
+        }
+        private bool IsNullable(Type paramType, out Type underlyingType)
+        {
+            underlyingType = Nullable.GetUnderlyingType(paramType);
+            return underlyingType != null;
+        }        
+
+        private ValidationError GetErrorOrDefault(Type parameterType)
+        {
+            switch (parameterType)
+            {
+                case Type t when IsNullable(t, out Type underType):
+                    return GetErrorOrDefault(underType);
+
+                case Type t when IsFlagsEnum(t):
+                    return new ValidationError(
+                        TreatsWarningsAsErrors,
+                        $"Unable to use {parameterType.Name} with [ParamsAllValues], because it's flags enum.");
+
+                case Type t when IsBool(t) || IsEnum(t):
+                    return default;
+
+                default:
+                    return new ValidationError(
+                        TreatsWarningsAsErrors,
+                        $"Type {parameterType.Name} cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type.");
+            }
+        }
+    }
+}
diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs
@@ -62,6 +62,7 @@ public IEnumerable<IValidator> GetValidators()
             yield return RunModeValidator.FailOnError;
             yield return GenericBenchmarksValidator.DontFailOnError;
             yield return DeferredExecutionValidator.FailOnError;
+            yield return ParamsAllValuesValidator.FailOnError;
         }
 
         public IEnumerable<Job> GetJobs() => Array.Empty<Job>();

diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs
@@ -161,30 +161,30 @@ private static Descriptor CreateDescriptor(
 
         private static ParameterDefinitions GetParameterDefinitions(Type type)
         {
-            const BindingFlags reflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
-
-            var allParamsMembers = type.GetTypeMembersWithGivenAttribute<ParamsAttribute>(reflectionFlags);
-
-            var allParamsSourceMembers = type.GetTypeMembersWithGivenAttribute<ParamsSourceAttribute>(reflectionFlags);
+            IEnumerable<ParameterDefinition> GetDefinitions<TAttribute>(Func<TAttribute, Type, object[]> getValidValues) where TAttribute : Attribute
+            {
+                const BindingFlags reflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
 
-            var definitions = allParamsMembers
-                .Select(member =>
+                var allMembers = type.GetTypeMembersWithGivenAttribute<TAttribute>(reflectionFlags);
+                return allMembers.Select(member =>
                     new ParameterDefinition(
                         member.Name,
                         member.IsStatic,
-                        GetValidValues(member.Attribute.Values, member.ParameterType),
-                        false))
-                .Concat(allParamsSourceMembers.Select(member =>
-                {
-                    var paramsValues = GetValidValuesForParamsSource(type, member.Attribute.Name);
-                    return new ParameterDefinition(
-                       member.Name,
-                       member.IsStatic,
-                       SmartParamBuilder.CreateForParams(paramsValues.source, paramsValues.values),
-                       false);
-                    
-                }))
-                .ToArray();
+                        getValidValues(member.Attribute, member.ParameterType),
+                        false));
+            }
+
+            var paramsDefinitions = GetDefinitions<ParamsAttribute>((attribute, parameterType) => GetValidValues(attribute.Values, parameterType));
+
+            var paramsSourceDefinitions = GetDefinitions<ParamsSourceAttribute>((attribute, _) =>
+            {
+                var paramsValues = GetValidValuesForParamsSource(type, attribute.Name);
+                return SmartParamBuilder.CreateForParams(paramsValues.source, paramsValues.values);
+            });
+
+            var paramsAllValuesDefinitions = GetDefinitions<ParamsAllValuesAttribute>((_, paramaterType) => GetAllValidValues(paramaterType));
+
+            var definitions = paramsDefinitions.Concat(paramsSourceDefinitions).Concat(paramsAllValuesDefinitions).ToArray();
 
             return new ParameterDefinitions(definitions);
         }
@@ -314,5 +314,25 @@ private static object[] ToArray(object sourceValue, MemberInfo memberInfo, Type
 
             return collection.Cast<object>().ToArray();
         }
+
+        private static object[] GetAllValidValues(Type parameterType)
+        {
+            if (parameterType == typeof(bool))
+                return new object[] { false, true };
+
+            if (parameterType.GetTypeInfo().IsEnum)
+            {
+                if (parameterType.GetTypeInfo().IsDefined(typeof(FlagsAttribute)))
+                    return new object[] { Activator.CreateInstance(parameterType) };
+
+                return Enum.GetValues(parameterType).Cast<object>().ToArray();
+            }
+
+            var nullableUnderlyingType = Nullable.GetUnderlyingType(parameterType);
+            if (nullableUnderlyingType != null)
+                return new object[] { null }.Concat(GetAllValidValues(nullableUnderlyingType)).ToArray();
+
+            return new object[] { Activator.CreateInstance(parameterType) };
+        }
     }
 }
diff --git a/...ramsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfBool.approved.txt b/...ramsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfBool.approved.txt
@@ -0,0 +1,14 @@
+=== WithAllValuesOfBool ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |         False | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |          True | 202.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 0
diff --git a/...ramsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfEnum.approved.txt b/...ramsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfEnum.approved.txt
@@ -0,0 +1,15 @@
+=== WithAllValuesOfEnum ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             A | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             B | 202.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             C | 302.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 0
diff --git a/...aluesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableBool.approved.txt b/...aluesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableBool.approved.txt
@@ -0,0 +1,15 @@
+=== WithAllValuesOfNullableBool ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             ? | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |         False | 202.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |          True | 302.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 0
diff --git a/...aluesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableEnum.approved.txt b/...aluesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableEnum.approved.txt
@@ -0,0 +1,16 @@
+=== WithAllValuesOfNullableEnum ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             ? | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             A | 202.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             B | 302.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             C | 402.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 0
diff --git a/...luesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedFlagsEnumError.approved.txt b/...luesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedFlagsEnumError.approved.txt
@@ -0,0 +1,14 @@
+=== WithNotAllowedFlagsEnumError ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             0 | 102.0 ns | 6.088 ns | 1.581 ns |
+
+Errors: 1
+* Unable to use TestFlagsEnum with [ParamsAllValues], because it's flags enum.
diff --git a/...sApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedNullableTypeError.approved.txt b/...sApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedNullableTypeError.approved.txt
@@ -0,0 +1,15 @@
+=== WithNotAllowedNullableTypeError ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             ? | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             0 | 202.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 1
+* Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type.
diff --git a/...AllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedTypeError.approved.txt b/...AllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedTypeError.approved.txt
@@ -0,0 +1,14 @@
+=== WithNotAllowedTypeError ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             0 | 102.0 ns | 6.088 ns | 1.581 ns |
+
+Errors: 1
+* Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type.