Adding tests for export of csv and csv Measurements

src/BenchmarkDotNet/Environments/EnvironmentResolver.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics.CodeAnalysis;
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Jobs;
@@ -9,10 +10,15 @@ namespace BenchmarkDotNet.Environments
 {
     public class EnvironmentResolver : Resolver
     {
-        public const int DefaultUnrollFactorForThroughput = 16;
-
-        public static readonly IResolver Instance = new CompositeResolver(new EnvironmentResolver(), GcResolver.Instance);
-
+        public const int DefaultUnrollFactorForThroughput = 16;
+
+        internal static readonly EnvironmentResolver Default = new EnvironmentResolver();
+
+        public static readonly IResolver Instance = new CompositeResolver(Default, GcResolver.Instance);
+        internal new void Register<[DynamicallyAccessedMembers(CharacteristicObject.CharacteristicMemberTypes)] T>(Characteristic<T> characteristic, Func<T> resolver)
+        {
+            base.Register(characteristic, resolver);
+        }
         private EnvironmentResolver()
         {
             Register(EnvironmentMode.PlatformCharacteristic, RuntimeInformation.GetCurrentPlatform);

tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs

@@ -1,13 +1,18 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Exporters.Csv;
 using BenchmarkDotNet.Exporters.Json;
 using BenchmarkDotNet.Exporters.Xml;
+using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Tests.Builders;
 using BenchmarkDotNet.Tests.Mocks;
@@ -43,15 +48,15 @@ public CommonExporterVerifyTests()
         public Task Exporters(string cultureInfoName)
         {
             var cultureInfo = CultureInfos[cultureInfoName];
-            Thread.CurrentThread.CurrentCulture = cultureInfo;
-
+            Thread.CurrentThread.CurrentCulture = cultureInfo;
+            EnvironmentResolver.Default.Register(EnvironmentMode.AffinityCharacteristic, () => new IntPtr(0xFF));
             var logger = new AccumulationLogger();
-
             var exporters = GetExporters();
             foreach (var exporter in exporters)
             {
-                PrintTitle(logger, exporter);
-                exporter.ExportToLog(MockFactory.CreateSummary(config.WithCultureInfo(cultureInfo)), logger);
+                PrintTitle(logger, exporter);
+                var summary = MockFactory.CreateSummary(config.WithCultureInfo(cultureInfo));
+                exporter.ExportToLog(summary, logger);
             }
 
             var settings = VerifySettingsFactory.Create();
@@ -92,8 +97,10 @@ private static IEnumerable<IExporter> GetExporters()
             yield return XmlExporter.BriefCompressed;
             yield return XmlExporter.Full;
             yield return XmlExporter.FullCompressed;
-        }
-
+            yield return CsvExporter.Default;
+            yield return CsvMeasurementsExporter.Default;
+        }
+
         private static readonly IConfig config = ManualConfig.Create(DefaultConfig.Instance)
             .AddColumn(StatisticColumn.Mean)
             .AddColumn(StatisticColumn.StdDev)

tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt

@@ -812,3 +812,15 @@ XmlExporter-full
 XmlExporter-full-compressed
 ############################################
 <?xml version="1.0" encoding="utf-8"?><Summary><Title>MockSummary</Title><HostEnvironmentInfo><BenchmarkDotNetCaption>BenchmarkDotNet</BenchmarkDotNetCaption><BenchmarkDotNetVersion>0.10.x-mock</BenchmarkDotNetVersion><OsVersion>Microsoft Windows NT 10.0.x.mock</OsVersion><ProcessorName>MockIntel Core i7-6700HQ CPU 2.60GHz</ProcessorName><PhysicalProcessorCount>1</PhysicalProcessorCount><PhysicalCoreCount>4</PhysicalCoreCount><LogicalCoreCount>8</LogicalCoreCount><RuntimeVersion>Clr 4.0.x.mock</RuntimeVersion><Architecture>64mock</Architecture><HasAttachedDebugger>False</HasAttachedDebugger><HasRyuJit>True</HasRyuJit><Configuration>CONFIGURATION</Configuration><ChronometerFrequency><Hertz>2531248</Hertz></ChronometerFrequency><HardwareTimerKind>Tsc</HardwareTimerKind></HostEnvironmentInfo><Benchmarks><BenchmarkCase><DisplayInfo>MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)</DisplayInfo><Namespace>BenchmarkDotNet.Tests.Mocks</Namespace><Type>MockBenchmarkClass</Type><Method>Foo</Method><MethodTitle>Foo</MethodTitle><Statistics><OriginalValues><Item>1</Item></OriginalValues><N>1</N><Min>1</Min><LowerFence>1</LowerFence><Q1>1</Q1><Median>1</Median><Mean>1</Mean><Q3>1</Q3><UpperFence>1</UpperFence><Max>1</Max><InterquartileRange>0</InterquartileRange><StandardError>0</StandardError><Variance>0</Variance><StandardDeviation>0</StandardDeviation><Skewness>NaN</Skewness><Kurtosis>NaN</Kurtosis><ConfidenceInterval><N>1</N><Mean>1</Mean><StandardError>0</StandardError><Level>L999</Level><Margin>NaN</Margin><Lower>NaN</Lower><Upper>NaN</Upper></ConfidenceInterval><Percentiles><P0>1</P0><P25>1</P25><P50>1</P50><P67>1</P67><P80>1</P80><P85>1</P85><P90>1</P90><P95>1</P95><P100>1</P100></Percentiles></Statistics><Memory><Gen0Collections>0</Gen0Collections><Gen1Collections>0</Gen1Collections><Gen2Collections>0</Gen2Collections><TotalOperations>0</TotalOperations><BytesAllocatedPerOperation /></Memory><Measurements><Measurement><IterationMode>Workload</IterationMode><IterationStage>Result</IterationStage><LaunchIndex>1</LaunchIndex><IterationIndex>1</IterationIndex><Operations>1</Operations><Nanoseconds>1</Nanoseconds></Measurement></Measurements></BenchmarkCase><BenchmarkCase><DisplayInfo>MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)</DisplayInfo><Namespace>BenchmarkDotNet.Tests.Mocks</Namespace><Type>MockBenchmarkClass</Type><Method>Bar</Method><MethodTitle>Bar</MethodTitle><Statistics><OriginalValues><Item>1</Item></OriginalValues><N>1</N><Min>1</Min><LowerFence>1</LowerFence><Q1>1</Q1><Median>1</Median><Mean>1</Mean><Q3>1</Q3><UpperFence>1</UpperFence><Max>1</Max><InterquartileRange>0</InterquartileRange><StandardError>0</StandardError><Variance>0</Variance><StandardDeviation>0</StandardDeviation><Skewness>NaN</Skewness><Kurtosis>NaN</Kurtosis><ConfidenceInterval><N>1</N><Mean>1</Mean><StandardError>0</StandardError><Level>L999</Level><Margin>NaN</Margin><Lower>NaN</Lower><Upper>NaN</Upper></ConfidenceInterval><Percentiles><P0>1</P0><P25>1</P25><P50>1</P50><P67>1</P67><P80>1</P80><P85>1</P85><P90>1</P90><P95>1</P95><P100>1</P100></Percentiles></Statistics><Memory><Gen0Collections>0</Gen0Collections><Gen1Collections>0</Gen1Collections><Gen2Collections>0</Gen2Collections><TotalOperations>0</TotalOperations><BytesAllocatedPerOperation /></Memory><Measurements><Measurement><IterationMode>Workload</IterationMode><IterationStage>Result</IterationStage><LaunchIndex>1</LaunchIndex><IterationIndex>1</IterationIndex><Operations>1</Operations><Nanoseconds>1</Nanoseconds></Measurement></Measurements></BenchmarkCase></Benchmarks></Summary>
+############################################
+CsvExporter
+############################################
+Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,LargeAddressAware,Platform,PowerPlanMode,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,NuGetReferences,Toolchain,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MemoryRandomization,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Mean,Error,StdDev,P67
+Foo,LongRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,16,15,1.000 ns,NA,0.000 ns,1.000 ns
+Bar,LongRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,16,15,1.000 ns,NA,0.000 ns,1.000 ns
+############################################
+CsvMeasurementsExporter
+############################################
+Target,Target_Namespace,Target_Type,Target_Method,Job_Id,Job_AnalyzeLaunchVariance,Job_EvaluateOverhead,Job_MaxAbsoluteError,Job_MaxRelativeError,Job_MinInvokeCount,Job_MinIterationTime,Job_OutlierMode,Job_Affinity,Job_EnvironmentVariables,Job_Jit,Job_LargeAddressAware,Job_Platform,Job_PowerPlanMode,Job_Runtime,Job_AllowVeryLargeObjects,Job_Concurrent,Job_CpuGroups,Job_Force,Job_HeapAffinitizeMask,Job_HeapCount,Job_NoAffinitize,Job_RetainVm,Job_Server,Job_Arguments,Job_BuildConfiguration,Job_Clock,Job_EngineFactory,Job_NuGetReferences,Job_Toolchain,Job_IsMutator,Job_InvocationCount,Job_IterationCount,Job_IterationTime,Job_LaunchCount,Job_MaxIterationCount,Job_MaxWarmupIterationCount,Job_MemoryRandomization,Job_MinIterationCount,Job_MinWarmupIterationCount,Job_RunStrategy,Job_UnrollFactor,Job_WarmupCount,Job_Display,Params,Measurement_LaunchIndex,Measurement_IterationMode,Measurement_IterationStage,Measurement_IterationIndex,Measurement_Nanoseconds,Measurement_Operations,Measurement_Value,Gen_0,Gen_1,Gen_2,Allocated_Bytes
+MockBenchmarkClass.Foo,BenchmarkDotNet.Tests.Mocks,MockBenchmarkClass,Foo,LongRun,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,Default,15,"LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)",,1,Workload,Result,1,1,1,1,0,0,0,?
+MockBenchmarkClass.Bar,BenchmarkDotNet.Tests.Mocks,MockBenchmarkClass,Bar,LongRun,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,Default,15,"LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)",,1,Workload,Result,1,1,1,1,0,0,0,?

tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt

@@ -812,3 +812,15 @@ XmlExporter-full
 XmlExporter-full-compressed
 ############################################
 <?xml version="1.0" encoding="utf-8"?><Summary><Title>MockSummary</Title><HostEnvironmentInfo><BenchmarkDotNetCaption>BenchmarkDotNet</BenchmarkDotNetCaption><BenchmarkDotNetVersion>0.10.x-mock</BenchmarkDotNetVersion><OsVersion>Microsoft Windows NT 10.0.x.mock</OsVersion><ProcessorName>MockIntel Core i7-6700HQ CPU 2.60GHz</ProcessorName><PhysicalProcessorCount>1</PhysicalProcessorCount><PhysicalCoreCount>4</PhysicalCoreCount><LogicalCoreCount>8</LogicalCoreCount><RuntimeVersion>Clr 4.0.x.mock</RuntimeVersion><Architecture>64mock</Architecture><HasAttachedDebugger>False</HasAttachedDebugger><HasRyuJit>True</HasRyuJit><Configuration>CONFIGURATION</Configuration><ChronometerFrequency><Hertz>2531248</Hertz></ChronometerFrequency><HardwareTimerKind>Tsc</HardwareTimerKind></HostEnvironmentInfo><Benchmarks><BenchmarkCase><DisplayInfo>MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)</DisplayInfo><Namespace>BenchmarkDotNet.Tests.Mocks</Namespace><Type>MockBenchmarkClass</Type><Method>Foo</Method><MethodTitle>Foo</MethodTitle><Statistics><OriginalValues><Item>1</Item></OriginalValues><N>1</N><Min>1</Min><LowerFence>1</LowerFence><Q1>1</Q1><Median>1</Median><Mean>1</Mean><Q3>1</Q3><UpperFence>1</UpperFence><Max>1</Max><InterquartileRange>0</InterquartileRange><StandardError>0</StandardError><Variance>0</Variance><StandardDeviation>0</StandardDeviation><Skewness>NaN</Skewness><Kurtosis>NaN</Kurtosis><ConfidenceInterval><N>1</N><Mean>1</Mean><StandardError>0</StandardError><Level>L999</Level><Margin>NaN</Margin><Lower>NaN</Lower><Upper>NaN</Upper></ConfidenceInterval><Percentiles><P0>1</P0><P25>1</P25><P50>1</P50><P67>1</P67><P80>1</P80><P85>1</P85><P90>1</P90><P95>1</P95><P100>1</P100></Percentiles></Statistics><Memory><Gen0Collections>0</Gen0Collections><Gen1Collections>0</Gen1Collections><Gen2Collections>0</Gen2Collections><TotalOperations>0</TotalOperations><BytesAllocatedPerOperation /></Memory><Measurements><Measurement><IterationMode>Workload</IterationMode><IterationStage>Result</IterationStage><LaunchIndex>1</LaunchIndex><IterationIndex>1</IterationIndex><Operations>1</Operations><Nanoseconds>1</Nanoseconds></Measurement></Measurements></BenchmarkCase><BenchmarkCase><DisplayInfo>MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)</DisplayInfo><Namespace>BenchmarkDotNet.Tests.Mocks</Namespace><Type>MockBenchmarkClass</Type><Method>Bar</Method><MethodTitle>Bar</MethodTitle><Statistics><OriginalValues><Item>1</Item></OriginalValues><N>1</N><Min>1</Min><LowerFence>1</LowerFence><Q1>1</Q1><Median>1</Median><Mean>1</Mean><Q3>1</Q3><UpperFence>1</UpperFence><Max>1</Max><InterquartileRange>0</InterquartileRange><StandardError>0</StandardError><Variance>0</Variance><StandardDeviation>0</StandardDeviation><Skewness>NaN</Skewness><Kurtosis>NaN</Kurtosis><ConfidenceInterval><N>1</N><Mean>1</Mean><StandardError>0</StandardError><Level>L999</Level><Margin>NaN</Margin><Lower>NaN</Lower><Upper>NaN</Upper></ConfidenceInterval><Percentiles><P0>1</P0><P25>1</P25><P50>1</P50><P67>1</P67><P80>1</P80><P85>1</P85><P90>1</P90><P95>1</P95><P100>1</P100></Percentiles></Statistics><Memory><Gen0Collections>0</Gen0Collections><Gen1Collections>0</Gen1Collections><Gen2Collections>0</Gen2Collections><TotalOperations>0</TotalOperations><BytesAllocatedPerOperation /></Memory><Measurements><Measurement><IterationMode>Workload</IterationMode><IterationStage>Result</IterationStage><LaunchIndex>1</LaunchIndex><IterationIndex>1</IterationIndex><Operations>1</Operations><Nanoseconds>1</Nanoseconds></Measurement></Measurements></BenchmarkCase></Benchmarks></Summary>
+############################################
+CsvExporter
+############################################
+Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,LargeAddressAware,Platform,PowerPlanMode,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,NuGetReferences,Toolchain,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MemoryRandomization,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Mean,Error,StdDev,P67
+Foo,LongRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,16,15,1.000 ns,NA,0.000 ns,1.000 ns
+Bar,LongRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,16,15,1.000 ns,NA,0.000 ns,1.000 ns
+############################################
+CsvMeasurementsExporter
+############################################
+Target,Target_Namespace,Target_Type,Target_Method,Job_Id,Job_AnalyzeLaunchVariance,Job_EvaluateOverhead,Job_MaxAbsoluteError,Job_MaxRelativeError,Job_MinInvokeCount,Job_MinIterationTime,Job_OutlierMode,Job_Affinity,Job_EnvironmentVariables,Job_Jit,Job_LargeAddressAware,Job_Platform,Job_PowerPlanMode,Job_Runtime,Job_AllowVeryLargeObjects,Job_Concurrent,Job_CpuGroups,Job_Force,Job_HeapAffinitizeMask,Job_HeapCount,Job_NoAffinitize,Job_RetainVm,Job_Server,Job_Arguments,Job_BuildConfiguration,Job_Clock,Job_EngineFactory,Job_NuGetReferences,Job_Toolchain,Job_IsMutator,Job_InvocationCount,Job_IterationCount,Job_IterationTime,Job_LaunchCount,Job_MaxIterationCount,Job_MaxWarmupIterationCount,Job_MemoryRandomization,Job_MinIterationCount,Job_MinWarmupIterationCount,Job_RunStrategy,Job_UnrollFactor,Job_WarmupCount,Job_Display,Params,Measurement_LaunchIndex,Measurement_IterationMode,Measurement_IterationStage,Measurement_IterationIndex,Measurement_Nanoseconds,Measurement_Operations,Measurement_Value,Gen_0,Gen_1,Gen_2,Allocated_Bytes
+MockBenchmarkClass.Foo,BenchmarkDotNet.Tests.Mocks,MockBenchmarkClass,Foo,LongRun,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,Default,15,"LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)",,1,Workload,Result,1,1,1,1,0,0,0,?
+MockBenchmarkClass.Bar,BenchmarkDotNet.Tests.Mocks,MockBenchmarkClass,Bar,LongRun,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,Default,15,"LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)",,1,Workload,Result,1,1,1,1,0,0,0,?