Merge pull request #2 from JanKrivanek/proto/embedded-content-handling

Embedded files support; determinism

.circleci/config.yml

@@ -9,6 +9,9 @@ jobs:
      docker:
       - image: mcr.microsoft.com/dotnet/sdk:8.0-preview
      steps:
+      - run:
+          name: install ssh and git
+          command: apt-get update && apt-get install -y openssh-client git
       - checkout
       - run: 
           name: Restore

NuGet.config

@@ -17,6 +17,11 @@
       <package pattern="Microsoft.Build.*" />
 	  <package pattern="Microsoft.Build" />
       <package pattern="Microsoft.NET.StringTools" />
+    </packageSource>
+	<packageSource key="offline-packages">
+      <package pattern="Microsoft.Build.*" />
+	  <package pattern="Microsoft.Build" />
+      <package pattern="Microsoft.NET.StringTools" />
     </packageSource>
   </packageSourceMapping>
 

Packages.props

@@ -1,6 +1,7 @@
 <Project>
   <Import Project="dependabot\Packages.props" />
   <ItemGroup>
-    <PackageVersion Include="Microsoft.Build" Version="17.8.0-preview-23424-02" />
+    <!-- <PackageVersion Include="Microsoft.Build" Version="17.8.0-preview-23424-02" /> -->
+	<PackageVersion Include="Microsoft.Build" Version="17.8.0-dev-23465-01" />
   </ItemGroup>
 </Project>

src/Microsoft.Build.BinlogRedactor/BinaryLog/IBinlogProcessor.cs

@@ -4,9 +4,13 @@ namespace Microsoft.Build.BinlogRedactor.BinaryLog;
 
 public interface IBinlogProcessor
 {
+    /// <summary>
+    /// Passes all string data from the input file to the <see cref="ISensitiveDataProcessor"/> and writes the
+    /// resulting binlog file so that only the appropriate textual data is altered, while result is still valid binlog.
+    /// </summary>
     Task<BinlogRedactorErrorCode> ProcessBinlog(
         string inputFileName,
         string outputFileName,
         ISensitiveDataProcessor sensitiveDataProcessor,
         CancellationToken cancellationToken);
-}
+}

src/Microsoft.Build.BinlogRedactor/BinaryLog/ISensitiveDataProcessor.cs

@@ -2,6 +2,9 @@
 
 public interface ISensitiveDataProcessor
 {
+    /// <summary>
+    /// Processes the given text and if needed, replaces sensitive data with a placeholder.
+    /// </summary>
     string ReplaceSensitiveData(string text);
     bool IsSensitiveData(string text);
 }

src/Microsoft.Build.BinlogRedactor/BinaryLog/SimpleBinlogProcessor.cs

@@ -14,20 +14,29 @@ public Task<BinlogRedactorErrorCode> ProcessBinlog(
         // Quick way:
         //
         BinaryLogReplayEventSource originalEventsSource = new BinaryLogReplayEventSource();
-        BinaryLogger bl = new BinaryLogger()
+        using BuildEventArgsReader originalBuildEventsReader =
+            BinaryLogReplayEventSource.OpenBuildEventsReader(inputFileName);
+        BinaryLogger outputBinlog = new BinaryLogger()
         {
-            Parameters = $"LogFile={outputFileName}",
+            Parameters = $"LogFile={outputFileName};ProjectImports=Replay;ReplayInitialInfo",
         };
-        bl.Initialize(originalEventsSource);
-        originalEventsSource.NotificationsSourceCreated += notifications => notifications.StringReadDone += args =>
-            args.StringToBeUsed = sensitiveDataProcessor.ReplaceSensitiveData(args.OriginalString);
-        originalEventsSource.Replay(inputFileName, cancellationToken);
+
+        originalBuildEventsReader.StringReadDone += HandleStringRead;
+        originalBuildEventsReader.ArchiveFileEncountered += ((Action<StringReadEventArgs>)HandleStringRead).ToArchiveFileHandler();
+
+        outputBinlog.Initialize(originalEventsSource);
+        originalEventsSource.Replay(originalBuildEventsReader, cancellationToken);
         cancellationToken.ThrowIfCancellationRequested();
-        bl.Shutdown();
+        outputBinlog.Shutdown();
 
         // TODO: error handling
 
         return Task.FromResult(BinlogRedactorErrorCode.Success);
+
+        void HandleStringRead(StringReadEventArgs args)
+        {
+            args.StringToBeUsed = sensitiveDataProcessor.ReplaceSensitiveData(args.OriginalString);
+        }
     }
 
 

test/Microsoft.Build.BinlogRedactor.Tests/BinlogRedactorTests.cs

@@ -1,19 +1,15 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using System.Runtime.CompilerServices;
 using FluentAssertions;
 using Microsoft.Build.BinlogRedactor.BinaryLog;
 using Microsoft.Build.BinlogRedactor.IO;
 using Microsoft.Build.BinlogRedactor.Reporting;
+using Microsoft.Build.Logging;
 using Microsoft.Extensions.Logging;
-using Xunit.Abstractions;
 
 namespace Microsoft.Build.BinlogRedactor.Tests
 {
     [UsesVerify]
-    public class BinlogRedactorTests: IClassFixture<TestLoggerFactory>
+    public class BinlogRedactorTests : IClassFixture<TestLoggerFactory>
     {
         private readonly ILoggerFactory _loggerFactory;
 
@@ -22,14 +18,63 @@ public BinlogRedactorTests(TestLoggerFactory testLoggerFactory)
             _loggerFactory = testLoggerFactory;
         }
 
+        static BinlogRedactorTests()
+        {
+            // TODO: this is a workaround until the MSBuild changes for reproducible binlogs
+            //  are published and we can use them in the tests (via dotnet sdk image).
+            // We have pre-created binlogs - but those are reproducible only for given OS/sdk
+            //  (mainly due to zipping).
+            ReplaceBinlogWithReplayed(Path.Combine("assets", "console.binlog"));
+            foreach (string binlogFile in GetBinlogFiles())
+            {
+                ReplaceBinlogWithReplayed(binlogFile);
+            }
+        }
+
+        private static void ReplaceBinlogWithReplayed(string binlogPath)
+        {
+            string replayedFile = binlogPath + "temp.binlog";
+
+            BinaryLogReplayEventSource replayEventSource = new BinaryLogReplayEventSource();
+            BuildEventArgsReader buildEventsReader =
+                BinaryLogReplayEventSource.OpenBuildEventsReader(binlogPath);
+            BinaryLogger outputBinlog = new BinaryLogger()
+            {
+                Parameters = $"LogFile={replayedFile};ProjectImports=Replay;ReplayInitialInfo",
+            };
+            // Subscribe empty action. But the mere subscribing forces unpacking and repacking of embedded files
+            buildEventsReader.ArchiveFileEncountered += arg => {};
+            outputBinlog.Initialize(replayEventSource);
+            replayEventSource.Replay(buildEventsReader, CancellationToken.None);
+            outputBinlog.Shutdown();
+            buildEventsReader.Dispose();
+
+            new PhysicalFileSystem().ReplaceFile(replayedFile, binlogPath);
+        }
+
+        private static IEnumerable<string> GetBinlogFiles([CallerFilePath] string sourceFile = "")
+        {
+            if (string.IsNullOrEmpty(sourceFile))
+            {
+                return Enumerable.Empty<string>();
+            }
+
+            string dir = Path.Combine(Path.GetDirectoryName(sourceFile)!, VerifyInitialization.SnapshotsDirectory);
+
+            return Directory.EnumerateFiles(dir, "*.binlog",
+                new EnumerationOptions() { IgnoreInaccessible = true, RecurseSubdirectories = true });
+        }
+
         [Fact]
         public async Task ExecuteIntegrationTest_NoOpRedactionShouldNotChangeFile()
         {
-            string outputFile = "output.binlog";
-            File.Delete(outputFile);
+            Environment.SetEnvironmentVariable("MSBUILDDETERMNISTICBINLOG", "1");
+
+            string outputFile = "console-redacted-01.binlog";
             string inputFile = Path.Combine("assets", "console.binlog");
 
             File.Exists(inputFile).Should().BeTrue();
+            File.Delete(outputFile);
 
             BinlogRedactorOptions options = new BinlogRedactorOptions(new string[] { Guid.NewGuid().ToString() })
             {
@@ -39,7 +84,9 @@ public async Task ExecuteIntegrationTest_NoOpRedactionShouldNotChangeFile()
             using FileDeletingScope fileDeletingScope = new FileDeletingScope(outputFile);
             BinlogRedactor binlogRedactor = new BinlogRedactor(_loggerFactory.CreateLogger<BinlogRedactor>(),
                 new PhysicalFileSystem(), new SimpleBinlogProcessor());
-            (await binlogRedactor.Execute(options).ConfigureAwait(false)).Should().Be(BinlogRedactorErrorCode.Success);
+
+            (await binlogRedactor.Execute(options).ConfigureAwait(false)).Should()
+                .Be(BinlogRedactorErrorCode.Success);
 
             File.Exists(outputFile).Should().BeTrue();
 
@@ -49,45 +96,47 @@ public async Task ExecuteIntegrationTest_NoOpRedactionShouldNotChangeFile()
 
         private static bool FilesAreBinaryEqual(FileInfo first, FileInfo second)
         {
-            if (first.Length != second.Length)
-                return false;
-
+            // Skipping shortcut test - so that we can better troubleshoot failures.
+            // if (first.Length != second.Length)
+            //    return false;
+
             if (string.Equals(first.FullName, second.FullName, StringComparison.OrdinalIgnoreCase))
                 return true;
 
             using FileStream fs1 = first.OpenRead();
             using FileStream fs2 = second.OpenRead();
             for (int i = 0; i < first.Length; i++)
             {
-                if (fs1.ReadByte() != fs2.ReadByte())
-                    return false;
+                byte b1 = (byte)fs1.ReadByte();
+                byte b2 = (byte)fs2.ReadByte();
+                if (b1 != b2)
+                    Assert.Fail($"Files ({first.Name}:{first.Length} and {second.Name}:{second.Length} sizes) are not equal at byte {i} ({b1} vs {b2})");
+                    //return false;
             }
-            
+
             return true;
         }
 
         [Fact]
         public async Task ExecuteIntegrationTest_RedactionShouldNotChangeOtherPartsOfFile()
         {
-            string outputFile = "output2.binlog";
+            string outputFile = "console-redacted-02.binlog";
             File.Delete(outputFile);
             string inputFile = Path.Combine("assets", "console.binlog");
 
             File.Exists(inputFile).Should().BeTrue();
 
             BinlogRedactorOptions options = new BinlogRedactorOptions(new string[] { "restore", "console" })
             {
-                InputPath = inputFile,
-                OutputFileName = outputFile,
-                OverWrite = false,
+                InputPath = inputFile, OutputFileName = outputFile, OverWrite = false,
             };
             using FileDeletingScope fileDeletingScope = new FileDeletingScope(outputFile);
-            BinlogRedactor binlogRedactor = new BinlogRedactor(_loggerFactory.CreateLogger<BinlogRedactor>(), new PhysicalFileSystem(), new SimpleBinlogProcessor());
+            BinlogRedactor binlogRedactor = new BinlogRedactor(_loggerFactory.CreateLogger<BinlogRedactor>(),
+                new PhysicalFileSystem(), new SimpleBinlogProcessor());
             (await binlogRedactor.Execute(options).ConfigureAwait(false)).Should().Be(BinlogRedactorErrorCode.Success);
 
             File.Exists(outputFile).Should().BeTrue();
 
-            // This is currently failing as the redaction is not deterministic - it produces slightly different output each time
             await VerifyFile(outputFile).ConfigureAwait(false);
         }
 

test/Microsoft.Build.BinlogRedactor.Tests/VerifyInitialization.cs

@@ -13,12 +13,14 @@ namespace Microsoft.Build.BinlogRedactor.Tests
 {
     internal static class VerifyInitialization
     {
+        internal const string SnapshotsDirectory = "Snapshots";
+
         [ModuleInitializer]
         public static void Initialize()
         {
             DerivePathInfo(
                 (_, _, type, method) => new(
-                    directory: "Snapshots",
+                    directory: SnapshotsDirectory,
                     typeName: type.Name,
                     methodName: method.Name));