extend ResultsComparer with what is needed to generate monthly perf r…

…eport (dotnet#2362)

* switch to System.CommandLine

* remove unused features

* extend DTOs with Metrics

* move common logic to Helper type

* move existing logic to TwoInputsComparer

* introduce MultipleInputsComparer that produces the matrix

* update docs

src/tools/ResultsComparer/DataTransferContracts.cs

@@ -5,7 +5,6 @@
 // <auto-generated />
 
 using System.Collections.Generic;
-using System.Linq;
 
 namespace DataTransferContracts // generated with http://json2csharp.com/#
 {
@@ -60,7 +59,7 @@ public class Percentiles
 
     public class Statistics
     {
-        public List<double> OriginalValues {  get; set; }
+        public double[] OriginalValues {  get; set; }
         public int N { get; set; }
         public double Min { get; set; }
         public double LowerFence { get; set; }
@@ -101,10 +100,28 @@ public class Measurement
         public double Nanoseconds { get; set; }
     }
 
+    public class Metric
+    {
+        public double Value { get; set; }
+        public MetricDescriptor Descriptor { get; set; }
+    }
+
+    public class MetricDescriptor
+    {
+        public string Id { get; set; }
+        public string DisplayName { get; set; }
+        public string Legend { get; set; }
+        public string NumberFormat { get; set; }
+        public int UnitType { get; set; }
+        public string Unit { get; set; }
+        public bool TheGreaterTheBetter { get; set; }
+        public int PriorityInCategory { get; set; }
+    }
+
     public class Benchmark
     {
         public string DisplayInfo { get; set; }
-        public object Namespace { get; set; }
+        public string Namespace { get; set; }
         public string Type { get; set; }
         public string Method { get; set; }
         public string MethodTitle { get; set; }
@@ -113,6 +130,7 @@ public class Benchmark
         public Statistics Statistics { get; set; }
         public Memory Memory { get; set; }
         public List<Measurement> Measurements { get; set; }
+        public List<Metric> Metrics { get; set; }
     }
 
     public class BdnResult

src/tools/ResultsComparer/Helper.cs

@@ -0,0 +1,55 @@
+using DataTransferContracts;
+using Newtonsoft.Json;
+using Perfolizer.Mathematics.Multimodality;
+using System;
+using System.IO;
+
+namespace ResultsComparer
+{
+    internal static class Helper
+    {
+        internal const string FullBdnJsonFileExtension = "full.json";
+
+        internal static string[] GetFilesToParse(string path)
+        {
+            if (Directory.Exists(path))
+                return Directory.GetFiles(path, $"*{Helper.FullBdnJsonFileExtension}", SearchOption.AllDirectories);
+            else if (File.Exists(path) || !path.EndsWith(Helper.FullBdnJsonFileExtension))
+                return new[] { path };
+            else
+                throw new FileNotFoundException($"Provided path does NOT exist or is not a {path} file", path);
+        }
+
+        // code and magic values taken from BenchmarkDotNet.Analysers.MultimodalDistributionAnalyzer
+        // See http://www.brendangregg.com/FrequencyTrails/modes.html
+        internal static string GetModalInfo(Benchmark benchmark)
+        {
+            if (benchmark.Statistics.N < 12) // not enough data to tell
+                return null;
+
+            double mValue = MValueCalculator.Calculate(benchmark.Statistics.OriginalValues);
+            if (mValue > 4.2)
+                return "multimodal";
+            else if (mValue > 3.2)
+                return "bimodal";
+            else if (mValue > 2.8)
+                return "several?";
+
+            return null;
+        }
+
+        internal static BdnResult ReadFromFile(string resultFilePath)
+        {
+            try
+            {
+                return JsonConvert.DeserializeObject<BdnResult>(File.ReadAllText(resultFilePath));
+            }
+            catch (JsonSerializationException)
+            {
+                Console.WriteLine($"Exception while reading the {resultFilePath} file.");
+
+                throw;
+            }
+        }
+    }
+}