dotnet#658: add tests and validator for ParamsAllValues

src/BenchmarkDotNet/Attributes/Validators/ParamsAllValuesValidator.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Extensions;
+
+namespace BenchmarkDotNet.Validators
+{
+    class ParamsAllValuesValidator : IValidator
+    {
+        public static readonly ParamsAllValuesValidator FailOnError = new ParamsAllValuesValidator();
+
+        private ParamsAllValuesValidator() { }
+
+        public bool TreatsWarningsAsErrors => true;
+
+        public IEnumerable<ValidationError> Validate(ValidationParameters input)
+        {
+            var validationErrors = new List<ValidationError>();
+
+            foreach (var groupByType in input.Benchmarks.GroupBy(benchmark => benchmark.Descriptor.Type))
+            {
+                const BindingFlags reflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
+                var allMembers = groupByType.Key.GetTypeMembersWithGivenAttribute<ParamsAllValuesAttribute>(reflectionFlags);
+                validationErrors.AddRange(allMembers.Select(member => member.ParameterType).SelectMany(ValidateAttibute));
+            }
+
+            return validationErrors;
+        }
+
+        private IEnumerable<ValidationError> ValidateAttibute(Type parameterType)
+        {
+            var nullableUnderlyingType = Nullable.GetUnderlyingType(parameterType);
+            var typeInfo = parameterType.GetTypeInfo();
+
+            if (typeInfo.IsEnum && typeInfo.IsDefined(typeof(FlagsAttribute)))
+            {
+                yield return new ValidationError(
+                    TreatsWarningsAsErrors, 
+                    $"Unable to use {parameterType.Name} with [ParamsAllValues], because it's flags enum.");
+            }
+            else if (nullableUnderlyingType != null)
+            {
+                foreach (var error in ValidateAttibute(nullableUnderlyingType))
+                {
+                    yield return error;
+                }
+            }
+            else if (parameterType != typeof(bool) && !typeInfo.IsEnum && nullableUnderlyingType == null)
+            {
+                yield return new ValidationError(
+                    TreatsWarningsAsErrors,
+                    $"Type {parameterType.Name} cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type.");
+            }
+        }
+    }
+}

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -59,6 +59,7 @@ public IEnumerable<IValidator> GetValidators()
             yield return RunModeValidator.FailOnError;
             yield return GenericBenchmarksValidator.DontFailOnError;
             yield return DeferredExecutionValidator.FailOnError;
+            yield return ParamsAllValuesValidator.FailOnError;
         }
 
         public IEnumerable<Job> GetJobs() => Array.Empty<Job>();

src/BenchmarkDotNet/Running/BenchmarkConverter.cs

@@ -323,7 +323,7 @@ private static object[] GetAllValidValues(Type parameterType)
             if (parameterType.GetTypeInfo().IsEnum)
             {
                 if (parameterType.GetTypeInfo().IsDefined(typeof(FlagsAttribute)))
-                    throw new InvalidOperationException($"Unable to use {parameterType.Name} with [ParamsAllValues], because it's flags enum.");
+                    return new object[] { Activator.CreateInstance(parameterType) };
 
                 return Enum.GetValues(parameterType).Cast<object>().ToArray();
             }
@@ -332,8 +332,7 @@ private static object[] GetAllValidValues(Type parameterType)
             if (nullableUnderlyingType != null)
                 return new object[] { null }.Concat(GetAllValidValues(nullableUnderlyingType)).ToArray();
 
-            throw new InvalidOperationException(
-                $"Type {parameterType.Name} cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type.");
+            return new object[] { Activator.CreateInstance(parameterType) };
         }
     }
 }

tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfBool.approved.txt

@@ -0,0 +1,14 @@
+=== WithAllValuesOfBool ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |         False | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |          True | 202.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 0

tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfEnum.approved.txt

@@ -0,0 +1,15 @@
+=== WithAllValuesOfEnum ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             A | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             B | 202.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             C | 302.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 0

tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableBool.approved.txt

@@ -0,0 +1,15 @@
+=== WithAllValuesOfNullableBool ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             ? | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |         False | 202.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |          True | 302.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 0

tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithAllValuesOfNullableEnum.approved.txt

@@ -0,0 +1,16 @@
+=== WithAllValuesOfNullableEnum ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             ? | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             A | 202.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             B | 302.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             C | 402.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 0

tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedFlagsEnumError.approved.txt

@@ -0,0 +1,14 @@
+=== WithNotAllowedFlagsEnumError ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             0 | 102.0 ns | 6.088 ns | 1.581 ns |
+
+Errors: 1
+* Unable to use TestFlagsEnum with [ParamsAllValues], because it's flags enum.

tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedNullableTypeError.approved.txt

@@ -0,0 +1,15 @@
+=== WithNotAllowedNullableTypeError ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             ? | 102.0 ns | 6.088 ns | 1.581 ns | ^
+ Benchmark |             0 | 202.0 ns | 6.088 ns | 1.581 ns | ^
+
+Errors: 1
+* Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type.

tests/BenchmarkDotNet.Tests/Attributes/ApprovedFiles/ParamsAllValuesApprovalTests.BenchmarkShouldProduceSummary.WithNotAllowedTypeError.approved.txt

@@ -0,0 +1,14 @@
+=== WithNotAllowedTypeError ===
+
+BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V
+MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
+Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
+
+
+    Method | ParamProperty |     Mean |    Error |   StdDev |
+---------- |-------------- |---------:|---------:|---------:|
+ Benchmark |             0 | 102.0 ns | 6.088 ns | 1.581 ns |
+
+Errors: 1
+* Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type.

tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesApprovalTests.cs

@@ -0,0 +1,148 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using ApprovalTests;
+using ApprovalTests.Namers;
+using ApprovalTests.Reporters;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Tests.Mocks;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Validators;
+using JetBrains.Annotations;
+using Xunit;
+using System.Collections.Generic;
+
+namespace BenchmarkDotNet.Tests.Attributes
+{
+    // In case of failed approval tests, use the following reporter:
+    // [UseReporter(typeof(KDiffReporter))]
+    [UseReporter(typeof(XUnit2Reporter))]
+    [UseApprovalSubdirectory("ApprovedFiles")]
+    [Collection("ApprovalTests")]
+    public class ParamsAllValuesApprovalTests : IDisposable
+    {
+        private readonly CultureInfo initCulture;
+
+        public ParamsAllValuesApprovalTests() => initCulture = Thread.CurrentThread.CurrentCulture;
+
+        [UsedImplicitly]
+        public static TheoryData<Type> GetBenchmarkTypes()
+        {
+            var data = new TheoryData<Type>();
+            foreach (var type in typeof(Benchmarks).GetNestedTypes())
+                data.Add(type);
+            return data;
+        }
+
+        [Theory]
+        [MemberData(nameof(GetBenchmarkTypes))]
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        public void BenchmarkShouldProduceSummary(Type benchmarkType)
+        {
+            NamerFactory.AdditionalInformation = benchmarkType.Name;
+            Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
+
+            var logger = new AccumulationLogger();
+            logger.WriteLine("=== " + benchmarkType.Name + " ===");
+
+            var exporter = MarkdownExporter.Mock;
+            var summary = MockFactory.CreateSummary(benchmarkType);
+            exporter.ExportToLog(summary, logger);
+
+            var validator = ParamsAllValuesValidator.FailOnError;
+            var errors = validator.Validate(new ValidationParameters(summary.BenchmarksCases, summary.Config)).ToList();
+            logger.WriteLine();
+            logger.WriteLine("Errors: " + errors.Count);
+            foreach (var error in errors)
+                logger.WriteLineError("* " + error.Message);
+
+            Approvals.Verify(logger.GetLog());
+        }
+
+        public void Dispose() => Thread.CurrentThread.CurrentCulture = initCulture;
+
+        public enum TestEnum
+        {
+            A = 1, B, C
+        }
+
+        [Flags]
+        public enum TestFlagsEnum
+        {
+            A = 0b001,
+            B = 0b010,
+            C = 0b100
+        }
+
+        [SuppressMessage("ReSharper", "InconsistentNaming")]
+        public static class Benchmarks
+        {
+            public class WithAllValuesOfBool
+            {
+                [ParamsAllValues]
+                public bool ParamProperty { get; set; }
+
+                [Benchmark]
+                public void Benchmark() { }
+            }
+
+            public class WithAllValuesOfEnum
+            {
+                [ParamsAllValues]
+                public TestEnum ParamProperty { get; set; }
+
+                [Benchmark]
+                public void Benchmark() { }
+            }
+
+            public class WithAllValuesOfNullableBool
+            {
+                [ParamsAllValues]
+                public bool? ParamProperty { get; set; }
+
+                [Benchmark]
+                public void Benchmark() { }
+            }
+
+            public class WithAllValuesOfNullableEnum
+            {
+                [ParamsAllValues]
+                public TestEnum? ParamProperty { get; set; }
+
+                [Benchmark]
+                public void Benchmark() { }
+            }
+
+            public class WithNotAllowedTypeError
+            {
+                [ParamsAllValues]
+                public int ParamProperty { get; set; }
+
+                [Benchmark]
+                public void Benchmark() { }
+            }
+
+            public class WithNotAllowedNullableTypeError
+            {
+                [ParamsAllValues]
+                public int? ParamProperty { get; set; }
+
+                [Benchmark]
+                public void Benchmark() { }
+            }
+
+            public class WithNotAllowedFlagsEnumError
+            {
+                [ParamsAllValues]
+                public TestFlagsEnum ParamProperty { get; set; }
+
+                [Benchmark]
+                public void Benchmark() { }
+            }
+        }
+    }
+}

tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj

@@ -40,4 +40,7 @@
   <ItemGroup>
     <EmbeddedResource Include="Portability\Cpu\TestFiles\*.txt" />
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Attributes\ApprovedFiles\" />
+  </ItemGroup>
 </Project>