Improve Disassembly exporters and add PrettyGithubMarkdownDiffDisassemblyExporter

src/BenchmarkDotNet/Diagnosers/DisassemblyDiagnoser.cs

@@ -37,7 +37,8 @@ private DisassemblyDiagnoser(WindowsDisassembler windowsDisassembler, MonoDisass
                 new CombinedDisassemblyExporter(results),
                 new RawDisassemblyExporter(results),
                 new PrettyHtmlDisassemblyExporter(results),
-                new PrettyGithubMarkdownDisassemblyExporter(results)
+                new PrettyGithubMarkdownDisassemblyExporter(results),
+                new PrettyGithubMarkdownDiffDisassemblyExporter(results)
             };
         }
 

src/BenchmarkDotNet/Exporters/PrettyGithubMarkdownDiffDisassemblyExporter.cs

@@ -0,0 +1,116 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Helpers;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using StreamWriter = BenchmarkDotNet.Portability.StreamWriter;
+
+namespace BenchmarkDotNet.Exporters
+{
+    public class PrettyGithubMarkdownDiffDisassemblyExporter : ExporterBase
+    {
+        private readonly IReadOnlyDictionary<BenchmarkCase, DisassemblyResult> results;
+
+        protected override string FileExtension => "md";
+        protected override string FileCaption => "asm.pretty.diff";
+
+        public PrettyGithubMarkdownDiffDisassemblyExporter(IReadOnlyDictionary<BenchmarkCase, DisassemblyResult> results) => this.results = results;
+
+        public override void ExportToLog(Summary summary, ILogger logger)
+        {
+            var benchmarksCases = summary.BenchmarksCases.Where(results.ContainsKey).ToArray();
+
+            logger.WriteLine($"## {summary.Title}");
+            for (int i = 0; i < benchmarksCases.Length; i++)
+            {
+                var firstBenchmarkCase = benchmarksCases[i];
+                for (int j = i + 1; j < benchmarksCases.Length; j++)
+                {
+                    var secondBenchmarkCase = benchmarksCases[j];
+
+                    ExportDiff(summary, logger, firstBenchmarkCase, secondBenchmarkCase);
+                }
+            }
+        }
+
+        private void ExportDiff(Summary summary, ILogger logger, BenchmarkCase firstBenchmarkCase, BenchmarkCase secondBenchmarkCase)
+        {
+            // We want to get diff for the same method and different JITs
+            if (firstBenchmarkCase.Descriptor.WorkloadMethod == secondBenchmarkCase.Descriptor.WorkloadMethod)
+            {
+                var firstFileName = SaveDisassemblyResult(summary, results[firstBenchmarkCase]);
+                var secondFileName = SaveDisassemblyResult(summary, results[secondBenchmarkCase]);
+                try
+                {
+                    var builder = new StringBuilder();
+
+                    RunGitDiff(firstFileName, secondFileName, builder);
+
+                    logger.WriteLine($"**Diff for {firstBenchmarkCase.Descriptor.WorkloadMethod.Name} method between:**");
+                    logger.WriteLine($"{GetImportantInfo(summary[firstBenchmarkCase])}");
+                    logger.WriteLine($"{GetImportantInfo(summary[secondBenchmarkCase])}");
+
+                    logger.WriteLine("```diff");
+                    logger.WriteLine(builder.ToString());
+                    logger.WriteLine("```");
+                }
+                finally
+                {
+                    File.Delete(firstFileName);
+                    File.Delete(secondFileName);
+                }
+            }
+        }
+
+        private static string SaveDisassemblyResult(Summary summary, DisassemblyResult disassemblyResult)
+        {
+            string filePath = $"{Path.Combine(summary.ResultsDirectoryPath, Guid.NewGuid().ToString())}-diff.temp";
+
+            if (File.Exists(filePath))
+                File.Delete(filePath);
+
+            using (var stream = StreamWriter.FromPath(filePath))
+            {
+                PrettyGithubMarkdownDisassemblyExporter.Export(new StreamLogger(stream), disassemblyResult, quotingCode: false);
+            }
+
+            return filePath;
+        }
+
+        private static string GetImportantInfo(BenchmarkReport benchmarkReport) => benchmarkReport.GetRuntimeInfo();
+
+        private static void RunGitDiff(string firstFile, string secondFile, StringBuilder result)
+        {
+            try
+            {
+                var output = ProcessHelper.RunAndReadOutputLineByLine("git", $"diff --no-index --no-color --text {firstFile} {secondFile}");
+
+                bool canRead = false;
+
+                foreach (string line in output)
+                {
+                    if (!string.IsNullOrEmpty(line) && line.Contains("@@"))
+                    {
+                        canRead = true;
+                        continue;
+                    }
+
+                    if (canRead)
+                    {
+                        result.AppendLine(line);
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                result.AppendLine("An exception occurred during run Git. Please check if you have Git installed on your system and Git is added to PATH.");
+                result.AppendLine($"Exception: {ex.Message}");
+            }
+        }
+    }
+}

src/BenchmarkDotNet/Exporters/PrettyGithubMarkdownDisassemblyExporter.cs

@@ -1,49 +1,43 @@
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
-using StreamWriter = BenchmarkDotNet.Portability.StreamWriter;
 
 namespace BenchmarkDotNet.Exporters
 {
-    public class PrettyGithubMarkdownDisassemblyExporter : IExporter
+    public class PrettyGithubMarkdownDisassemblyExporter : ExporterBase
     {
         private readonly IReadOnlyDictionary<BenchmarkCase, DisassemblyResult> results;
 
-        public PrettyGithubMarkdownDisassemblyExporter(IReadOnlyDictionary<BenchmarkCase, DisassemblyResult> results) => this.results = results;
-
-        public string Name => nameof(PrettyGithubMarkdownDisassemblyExporter);
-
-        public void ExportToLog(Summary summary, ILogger logger) { }
+        public PrettyGithubMarkdownDisassemblyExporter(IReadOnlyDictionary<BenchmarkCase, DisassemblyResult> results)
+        {
+            this.results = results;
+        }
 
-        public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
-            => summary.BenchmarksCases
-                      .Where(results.ContainsKey)
-                      .Select(benchmark => Export(summary, benchmark));
+        protected override string FileExtension => "md";
+        protected override string FileCaption => "asm.pretty";
 
-        private string Export(Summary summary, BenchmarkCase benchmarkCase)
+        public override void ExportToLog(Summary summary, ILogger logger)
         {
-            string filePath = $"{Path.Combine(summary.ResultsDirectoryPath, benchmarkCase.FolderInfo)}-asm.pretty.md";
-            if (File.Exists(filePath))
-                File.Delete(filePath);
-
-            using (var stream = StreamWriter.FromPath(filePath))
+            foreach (var benchmarkCase in summary.BenchmarksCases.Where(results.ContainsKey))
             {
-                Export(new StreamLogger(stream), results[benchmarkCase], benchmarkCase);
+                logger.WriteLine($"## {summary[benchmarkCase].GetRuntimeInfo()}");
+                Export(logger, results[benchmarkCase]);
             }
-
-            return filePath;
         }
 
-        private static void Export(ILogger logger, DisassemblyResult disassemblyResult, BenchmarkCase benchmarkCase)
+        internal static void Export(ILogger logger, DisassemblyResult disassemblyResult, bool quotingCode = true)
         {
             int methodIndex = 0;
             foreach (var method in disassemblyResult.Methods.Where(method => string.IsNullOrEmpty(method.Problem)))
             {
-                logger.WriteLine("```assembly");
+                if (quotingCode)
+                {
+                    logger.WriteLine("```assembly");
+                }
+
                 logger.WriteLine($"; {method.Name}");
 
                 var pretty = DisassemblyPrettifier.Prettify(method, $"M{methodIndex++:00}");
@@ -54,19 +48,22 @@ private static void Export(ILogger logger, DisassemblyResult disassemblyResult,
                     if (element is DisassemblyPrettifier.Label label)
                     {
                         logger.WriteLine($"{label.TextRepresentation}:");
-                        
+
                         continue;
                     }
                     if (element.Source is Asm asm)
                     {
                         totalSizeInBytes += asm.SizeInBytes;
                     }
-                    
-                    logger.WriteLine($"       {element.TextRepresentation}");                        
+
+                    logger.WriteLine($"       {element.TextRepresentation}");
                 }
 
                 logger.WriteLine($"; Total bytes of code {totalSizeInBytes}");
-                logger.WriteLine("```");
+                if (quotingCode)
+                {
+                    logger.WriteLine("```");
+                }
             }
 
             foreach (var withProblems in disassemblyResult.Methods
@@ -79,6 +76,8 @@ private static void Export(ILogger logger, DisassemblyResult disassemblyResult,
                     logger.WriteLine(withProblem.Name);
                 }
             }
+
+            logger.WriteLine();
         }
     }
 }

src/BenchmarkDotNet/Exporters/PrettyHtmlDisassemblyExporter.cs

@@ -1,64 +1,57 @@
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Helpers;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
-using StreamWriter = BenchmarkDotNet.Portability.StreamWriter;
 
 namespace BenchmarkDotNet.Exporters
 {
-    public class PrettyHtmlDisassemblyExporter : IExporter
+    public class PrettyHtmlDisassemblyExporter : ExporterBase
     {
         private static readonly Lazy<string> HighlightingLabelsScript = new Lazy<string>(() => ResourceHelper.LoadTemplate("highlightingLabelsScript.js"));
-
+            
         private readonly IReadOnlyDictionary<BenchmarkCase, DisassemblyResult> results;
 
         public PrettyHtmlDisassemblyExporter(IReadOnlyDictionary<BenchmarkCase, DisassemblyResult> results) => this.results = results;
 
-        public string Name => nameof(PrettyHtmlDisassemblyExporter);
-
-        public void ExportToLog(Summary summary, ILogger logger) { }
-
-        public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
-            => summary.BenchmarksCases
-                      .Where(results.ContainsKey)
-                      .Select(benchmark => Export(summary, benchmark));
-
-        private string Export(Summary summary, BenchmarkCase benchmarkCase)
-        {
-            string filePath = $"{Path.Combine(summary.ResultsDirectoryPath, benchmarkCase.FolderInfo)}-asm.pretty.html";
-            if (File.Exists(filePath))
-                File.Delete(filePath);
-
-            using (var stream = StreamWriter.FromPath(filePath))
-            {
-                Export(new StreamLogger(stream), results[benchmarkCase], benchmarkCase);
-            }
-
-            return filePath;
-        }
+        protected override string FileExtension => "html";
+        protected override string FileCaption => "asm.pretty";
 
-        private static void Export(ILogger logger, DisassemblyResult disassemblyResult, BenchmarkCase benchmarkCase)
+        public override void ExportToLog(Summary summary, ILogger logger)
         {
             logger.WriteLine("<!DOCTYPE html><html lang='en'><head><meta charset='utf-8' /><head>");
-            logger.WriteLine($"<title>Pretty Output of DisassemblyDiagnoser for {benchmarkCase.DisplayInfo}</title>");
+            logger.WriteLine($"<title>Pretty Output of DisassemblyDiagnoser for {summary.Title}</title>");
             logger.WriteLine(InstructionPointerExporter.CssStyle);
             logger.WriteLine(@"
 <style type='text/css'>
     td.label:hover { cursor: pointer; background-color: yellow !important; }
     td.highlighted { background-color: yellow !important; }
-</style>");
+</style></head><body>");
             logger.WriteLine("<script src=\"https://ajax.aspnetcdn.com/ajax/jQuery/jquery-3.2.1.min.js\"></script>");
             logger.WriteLine($"<script>{HighlightingLabelsScript.Value}</script>");
-            logger.WriteLine("</head><body><table><tbody>");
 
-            int methodIndex = 0, referenceIndex = 0;
+            int referenceIndex = 0;
+
+            foreach (var benchmarkCase in summary.BenchmarksCases.Where(results.ContainsKey))
+            {
+                Export(logger, summary, results[benchmarkCase], benchmarkCase, ref referenceIndex);
+            }
+
+            logger.WriteLine("</body></html>");
+        }
+
+        private static void Export(ILogger logger, Summary summary, DisassemblyResult disassemblyResult, BenchmarkCase benchmarkCase, ref int referenceIndex)
+        {
+            logger.WriteLine($"<h2>{summary[benchmarkCase].GetRuntimeInfo()}</h2>");
+            logger.WriteLine("<table><tbody>");
+
+            int methodIndex = 0;
             foreach (var method in disassemblyResult.Methods.Where(method => string.IsNullOrEmpty(method.Problem)))
             {
+                referenceIndex++;
                 logger.WriteLine($"<tr><th colspan=\"2\" style=\"text-align: left;\">{method.Name}</th><th></th></tr>");
 
                 var pretty = DisassemblyPrettifier.Prettify(method, $"M{methodIndex++:00}");
@@ -71,7 +64,7 @@ private static void Export(ILogger logger, DisassemblyResult disassemblyResult,
                         even = !even;
 
                         logger.WriteLine($"<tr class=\"{(even && diffTheLabels ? "evenMap" : string.Empty)}\">");
-                        logger.WriteLine($"<td id=\"{label.Id}\" class=\"label\" data-label=\"{label.TextRepresentation}\"><pre><code>{label.TextRepresentation}</pre></code></td>");
+                        logger.WriteLine($"<td id=\"{referenceIndex}_{label.Id}\" class=\"label\" data-label=\"{referenceIndex}_{label.TextRepresentation}\"><pre><code>{label.TextRepresentation}</pre></code></td>");
                         logger.WriteLine("<td>&nbsp;</td></tr>");
 
                         continue;
@@ -83,7 +76,7 @@ private static void Export(ILogger logger, DisassemblyResult disassemblyResult,
                     string tooltip = element.Source is Asm asm ? $"title=\"{asm.TextRepresentation}\"" : null;
 
                     if (element is DisassemblyPrettifier.Reference reference)
-                        logger.WriteLine($"<td id=\"{referenceIndex++}\" class=\"reference\" data-reference=\"{reference.Id}\" {tooltip}><a href=\"#{reference.Id}\"><pre><code>{reference.TextRepresentation}</pre></code></a></td>");
+                        logger.WriteLine($"<td id=\"{referenceIndex}\" class=\"reference\" data-reference=\"{referenceIndex}_{reference.Id}\" {tooltip}><a href=\"#{referenceIndex}_{reference.Id}\"><pre><code>{reference.TextRepresentation}</pre></code></a></td>");
                     else
                         logger.WriteLine($"<td {tooltip}><pre><code>{element.TextRepresentation}</pre></code></td>");
 
@@ -105,7 +98,7 @@ private static void Export(ILogger logger, DisassemblyResult disassemblyResult,
                 logger.WriteLine("<tr><td colspan=\"{2}\"></td></tr>");
             }
 
-            logger.WriteLine("</tbody></table></body></html>");
+            logger.WriteLine("</tbody></table>");
         }
     }
 }