From b8e30485eb68e31688e0149c57a7fa335e34860b Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 Sep 2023 13:38:28 +0200 Subject: [PATCH 1/8] Add embedded content transfering and build events binary equality --- .../BinaryLogger/BinaryLogRecordKind.cs | 2 +- .../BinaryLogReplayEventSource.cs | 66 ++++++++++----- .../Logging/BinaryLogger/BinaryLogger.cs | 35 ++++++-- .../BinaryLogger/BuildEventArgsReader.cs | 60 +++++++++++--- .../BinaryLogger/BuildEventArgsWriter.cs | 73 ++++++++-------- .../Postprocessing/ArchiveFile.cs | 78 +++++++++++++++++ .../EmbeddedContentEventArgs.cs | 22 +++++ .../Postprocessing/EmbeddedContentKind.cs | 17 ++++ .../EmbeddedContentKindExtensions.cs | 22 +++++ .../Postprocessing/GreedyBufferedStream.cs | 83 +++++++++++++++++++ .../IBuildEventArgsReaderNotifications.cs | 2 +- .../IBuildEventStringsReader.cs | 0 .../Postprocessing/IEmbeddedContentSource.cs | 13 +++ .../StringReadEventArgs.cs | 0 src/Build/Microsoft.Build.csproj | 12 ++- src/Framework/Traits.cs | 6 ++ 16 files changed, 414 insertions(+), 77 deletions(-) create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs rename src/Build/Logging/BinaryLogger/{ => Postprocessing}/IBuildEventArgsReaderNotifications.cs (89%) rename src/Build/Logging/BinaryLogger/{ => Postprocessing}/IBuildEventStringsReader.cs (100%) create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs rename src/Build/Logging/BinaryLogger/{ => Postprocessing}/StringReadEventArgs.cs (100%) diff --git a/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs b/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs index 9fe1638fd3a..28333110721 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs @@ -24,7 +24,7 @@ internal enum BinaryLogRecordKind ProjectEvaluationStarted, ProjectEvaluationFinished, ProjectImported, - ProjectImportArchive, + ProjectImportArchive = 17, TargetSkipped, PropertyReassignment, UninitializedPropertyRead, diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index a19a06c2d37..b21169ed135 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -26,11 +26,6 @@ static BinaryLogReplayEventSource() _ = ItemGroupLoggingHelper.ItemGroupIncludeLogMessagePrefix; } - /// - /// Raised once is created during replaying - /// - public event Action? NotificationsSourceCreated; - /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// @@ -68,6 +63,38 @@ public static BinaryReader OpenReader(string sourceFilePath) } } + /// + /// Creates a for the provided binary reader over binary log file. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// Indicates whether the passed BinaryReader should be closed on disposing. + /// BuildEventArgsReader over the given binlog file binary reader. + public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryReader, bool closeInput) + { + int fileFormatVersion = binaryReader.ReadInt32(); + + // the log file is written using a newer version of file format + // that we don't know how to read + if (fileFormatVersion > BinaryLogger.FileFormatVersion) + { + var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); + throw new NotSupportedException(text); + } + + return new BuildEventArgsReader(binaryReader, fileFormatVersion) { CloseInput = closeInput }; + } + + /// + /// Creates a for the provided binary log file. + /// Performs decompression and buffering in the optimal way. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// BinaryReader of the given binlog file. + public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) + => OpenBuildEventsReader(OpenReader(sourceFilePath), true); + /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// @@ -75,30 +102,29 @@ public static BinaryReader OpenReader(string sourceFilePath) /// A indicating the replay should stop as soon as possible. public void Replay(string sourceFilePath, CancellationToken cancellationToken) { - using var binaryReader = OpenReader(sourceFilePath); - Replay(binaryReader, cancellationToken); + using var eventsReader = OpenBuildEventsReader(sourceFilePath); + Replay(eventsReader, cancellationToken); } /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// /// The binary log content binary reader - caller is responsible for disposing. + /// Indicates whether the passed BinaryReader should be closed on disposing. /// A indicating the replay should stop as soon as possible. - public void Replay(BinaryReader binaryReader, CancellationToken cancellationToken) + public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken cancellationToken) { - int fileFormatVersion = binaryReader.ReadInt32(); - - // the log file is written using a newer version of file format - // that we don't know how to read - if (fileFormatVersion > BinaryLogger.FileFormatVersion) - { - var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); - throw new NotSupportedException(text); - } - - using var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion); - NotificationsSourceCreated?.Invoke(reader); + using var reader = OpenBuildEventsReader(binaryReader, closeInput); + Replay(reader, cancellationToken); + } + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The build events reader - caller is responsible for disposing. + /// A indicating the replay should stop as soon as possible. + public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken) + { while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) { Dispatch(instance); diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index fb8e59007e3..a95480428dd 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -61,7 +61,10 @@ public sealed class BinaryLogger : ILogger // - new record kind: ResponseFileUsedEventArgs // version 16: // - AssemblyLoadBuildEventArgs - internal const int FileFormatVersion = 16; + // version 17: + // - Making ProjectStartedEventArgs, ProjectEvaluationFinishedEventArgs, AssemblyLoadBuildEventArgs equal + // between de/serialization roundtrips. + internal const int FileFormatVersion = 17; private Stream stream; private BinaryWriter binaryWriter; @@ -115,7 +118,12 @@ public enum ProjectImportsCollectionMode /// /// Initializes the logger by subscribing to events of the specified event source. /// - public void Initialize(IEventSource eventSource) + public void Initialize(IEventSource eventSource) => Initialize(eventSource, null); + + /// + /// Initializes the logger by subscribing to events of the specified event source and embedded content source. + /// + public void Initialize(IEventSource eventSource, IEmbeddedContentSource embeddedFilesSource) { _initialTargetOutputLogging = Environment.GetEnvironmentVariable("MSBUILDTARGETOUTPUTLOGGING"); _initialLogImports = Traits.Instance.EscapeHatches.LogProjectImports; @@ -130,6 +138,13 @@ public void Initialize(IEventSource eventSource) ProcessParameters(); + if (embeddedFilesSource != null) + { + CollectProjectImports = ProjectImportsCollectionMode.None; + embeddedFilesSource.EmbeddedContentRead += args => + eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream, args.Length); + } + try { string logDirectory = null; @@ -178,7 +193,9 @@ public void Initialize(IEventSource eventSource) // wrapping the GZipStream in a buffered stream significantly improves performance // and the max throughput is reached with a 32K buffer. See details here: // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 - stream = new BufferedStream(stream, bufferSize: 32768); + stream = Traits.Instance.DeterministicBinlogStreamBuffering ? + new GreedyBufferedStream(stream, bufferSize: 32768) : + new BufferedStream(stream, bufferSize: 32768); binaryWriter = new BinaryWriter(stream); eventArgsWriter = new BuildEventArgsWriter(binaryWriter); @@ -187,9 +204,15 @@ public void Initialize(IEventSource eventSource) eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile; } - binaryWriter.Write(FileFormatVersion); - - LogInitialInfo(); + if (embeddedFilesSource == null) + { + binaryWriter.Write(FileFormatVersion); + LogInitialInfo(); + } + else + { + binaryWriter.Write(embeddedFilesSource.FileFormatVersion); + } eventSource.AnyEventRaised += EventSource_AnyEventRaised; diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 4b13c438721..4af1ecd28b8 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -63,9 +63,19 @@ public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) this.fileFormatVersion = fileFormatVersion; } + /// + /// Directs whether the passed should be closed when this instance is disposed. + /// Defaults to "false". + /// + public bool CloseInput { private get; set; } = false; + public void Dispose() { stringStorage.Dispose(); + if (CloseInput) + { + binaryReader.Dispose(); + } } /// @@ -81,11 +91,14 @@ public void Dispose() /// public event Action? StringEncountered; + public int FileFormatVersion => fileFormatVersion; + /// - /// Raised when the log reader encounters a binary blob embedded in the stream. - /// The arguments include the blob kind and the byte buffer with the contents. + /// Raised when the log reader encounters a project import archive (embedded content) in the stream. + /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. + /// If no subscriber is attached, the data is skipped. /// - internal event Action? OnBlobRead; + public event Action? EmbeddedContentRead; /// /// Reads the next log record from the . @@ -113,7 +126,7 @@ public void Dispose() } else if (recordKind == BinaryLogRecordKind.ProjectImportArchive) { - ReadBlob(recordKind); + ReadEmbeddedContent(recordKind); } recordNumber += 1; @@ -214,11 +227,30 @@ private static bool IsAuxiliaryRecord(BinaryLogRecordKind recordKind) || recordKind == BinaryLogRecordKind.ProjectImportArchive; } - private void ReadBlob(BinaryLogRecordKind kind) + private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) { int length = ReadInt32(); - byte[] bytes = binaryReader.ReadBytes(length); - OnBlobRead?.Invoke(kind, bytes); + if (EmbeddedContentRead != null) + { + long preEventPosition = binaryReader.BaseStream.CanSeek ? binaryReader.BaseStream.Position : 0; + EmbeddedContentRead(new EmbeddedContentEventArgs(recordKind.ToEmbeddedContentKind(), binaryReader.BaseStream, length)); + long postEventPosition = binaryReader.BaseStream.CanSeek ? binaryReader.BaseStream.Position : length; + if (postEventPosition - preEventPosition != length) + { + throw new InvalidDataException($"The {nameof(EmbeddedContentRead)} event handler must read exactly {length} bytes from the stream."); + } + } + else + { + if (binaryReader.BaseStream.CanSeek) + { + binaryReader.BaseStream.Seek(length, SeekOrigin.Current); + } + else + { + binaryReader.ReadBytes(length); + } + } } private void ReadNameValueList() @@ -421,11 +453,12 @@ private BuildEventArgs ReadProjectEvaluationFinishedEventArgs() if (fileFormatVersion >= 12) { - IEnumerable? globalProperties = null; - if (ReadBoolean()) + if (fileFormatVersion < 17) { - globalProperties = ReadStringDictionary(); + // Throw away, but need to advance past it + ReadBoolean(); } + IEnumerable? globalProperties = ReadStringDictionary(); var propertyList = ReadPropertyList(); var itemList = ReadProjectItems(); @@ -476,10 +509,12 @@ private BuildEventArgs ReadProjectStartedEventArgs() if (fileFormatVersion > 6) { - if (ReadBoolean()) + if (fileFormatVersion < 17) { - globalProperties = ReadStringDictionary(); + // Throw away, but need to advance past it + ReadBoolean(); } + globalProperties = ReadStringDictionary(); } var propertyList = ReadPropertyList(); @@ -835,6 +870,7 @@ private AssemblyLoadBuildEventArgs ReadAssemblyLoadEventArgs() mvid, appDomainName); SetCommonFields(e, fields); + e.ProjectFile = fields.ProjectFile; return e; } diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs index bc96814843e..3d1e9258e75 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Globalization; @@ -210,20 +211,15 @@ private void WriteCore(BuildEventArgs e) } } - public void WriteBlob(BinaryLogRecordKind kind, byte[] bytes) - { - // write the blob directly to the underlying writer, - // bypassing the memory stream - using var redirection = RedirectWritesToOriginalWriter(); - - Write(kind); - Write(bytes.Length); - Write(bytes); - } + public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int length) + => WriteBlobImpl(kind, stream, length); public void WriteBlob(BinaryLogRecordKind kind, Stream stream) + => WriteBlobImpl(kind, stream, null); + + private void WriteBlobImpl(BinaryLogRecordKind kind, Stream stream, int? length) { - if (stream.Length > int.MaxValue) + if (stream.CanSeek && stream.Length > int.MaxValue) { throw new ArgumentOutOfRangeException(nameof(stream)); } @@ -233,8 +229,8 @@ public void WriteBlob(BinaryLogRecordKind kind, Stream stream) using var redirection = RedirectWritesToOriginalWriter(); Write(kind); - Write((int)stream.Length); - Write(stream); + Write(length ?? (int)stream.Length); + Write(stream, length); } /// @@ -298,15 +294,7 @@ private void Write(ProjectEvaluationFinishedEventArgs e) WriteBuildEventArgsFields(e, writeMessage: false); WriteDeduplicatedString(e.ProjectFile); - if (e.GlobalProperties == null) - { - Write(false); - } - else - { - Write(true); - WriteProperties(e.GlobalProperties); - } + WriteProperties(e.GlobalProperties); WriteProperties(e.Properties); @@ -347,15 +335,7 @@ private void Write(ProjectStartedEventArgs e) WriteDeduplicatedString(e.TargetNames); WriteDeduplicatedString(e.ToolsVersion); - if (e.GlobalProperties == null) - { - Write(false); - } - else - { - Write(true); - Write(e.GlobalProperties); - } + Write(e.GlobalProperties); WriteProperties(e.Properties); @@ -1095,7 +1075,7 @@ private void Write(BinaryLogRecordKind kind) Write((int)kind); } - private void Write(int value) + internal void Write(int value) { BinaryWriterExtensions.Write7BitEncodedInt(binaryWriter, value); } @@ -1110,9 +1090,34 @@ private void Write(byte[] bytes) binaryWriter.Write(bytes); } - private void Write(Stream stream) + private void Write(Stream stream, int? length) { - stream.CopyTo(binaryWriter.BaseStream); + if (length == null) + { + stream.CopyTo(binaryWriter.BaseStream); + return; + } + + // borrowed from runtime from Stream.cs + const int defaultCopyBufferSize = 81920; + int bufferSize = Math.Min(defaultCopyBufferSize, length.Value); + + byte[] buffer = ArrayPool.Shared.Rent(bufferSize); + try + { + int bytesRead; + while ( + length > 0 && + (bytesRead = stream.Read(buffer, 0, Math.Min(buffer.Length, length.Value))) != 0) + { + binaryWriter.BaseStream.Write(buffer, 0, bytesRead); + length -= bytesRead; + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } } private void Write(byte b) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs new file mode 100644 index 00000000000..59868eb43dd --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.IO.Compression; + +namespace Microsoft.Build.Logging +{ + public class ArchiveFile : IDisposable + { + public ArchiveFile(string fullPath, StreamReader contentReader) + { + FullPath = fullPath; + _contentReader = contentReader; + } + + public static ArchiveFile From(ZipArchiveEntry entry) + { + return new ArchiveFile(CalculateArchivePath(entry.FullName), new StreamReader(entry.Open())); + } + + public string FullPath { get; } + + public StreamReader GetContentReader() + { + if (_stringAcquired) + { + throw new InvalidOperationException("Content already acquired as string via GetContent."); + } + + _streamAcquired = true; + return _contentReader; + } + + public string GetContent() + { + if (_streamAcquired) + { + throw new InvalidOperationException("Content already acquired as StreamReader via GetContnetReader."); + } + + if (!_stringAcquired) + { + _stringAcquired = true; + _content = _contentReader.ReadToEnd(); + } + + return _content!; + } + + private bool _streamAcquired; + private bool _stringAcquired; + private readonly StreamReader _contentReader; + private string? _content; + + public static string CalculateArchivePath(string filePath) + { + string archivePath = filePath; + + if (filePath.Contains(":") || (!filePath.StartsWith("\\") && !filePath.StartsWith("/"))) + { + archivePath = archivePath.Replace(":", ""); + archivePath = archivePath.Replace("/", "\\"); + archivePath = archivePath.Replace("\\\\", "\\"); + } + else + { + archivePath = archivePath.Replace("\\", "/"); + archivePath = archivePath.Replace("//", "/"); + } + + return archivePath; + } + + public void Dispose() => _contentReader.Dispose(); + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs new file mode 100644 index 00000000000..b5d6e985783 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging +{ + public sealed class EmbeddedContentEventArgs : EventArgs + { + public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream, int length) + { + ContentKind = contentKind; + ContentStream = contentStream; + Length = length; + } + + public EmbeddedContentKind ContentKind { get; } + public Stream ContentStream { get; } + public int Length { get; } + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs new file mode 100644 index 00000000000..7f694c91363 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Build.Logging +{ + public enum EmbeddedContentKind + { + Unknown = -1, + ProjectImportArchive = 17, + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs new file mode 100644 index 00000000000..73e9251cd77 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Build.Logging +{ + internal static class EmbeddedContentKindExtensions + { + internal static EmbeddedContentKind ToEmbeddedContentKind(this BinaryLogRecordKind kind) + { + return kind == BinaryLogRecordKind.ProjectImportArchive + ? EmbeddedContentKind.ProjectImportArchive + : EmbeddedContentKind.Unknown; + } + + internal static BinaryLogRecordKind ToBinaryLogRecordKind(this EmbeddedContentKind kind) + { + return kind == EmbeddedContentKind.ProjectImportArchive + ? BinaryLogRecordKind.ProjectImportArchive + : (BinaryLogRecordKind)EmbeddedContentKind.Unknown; + } + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs new file mode 100644 index 00000000000..e334eac4b8f --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging +{ + /// + /// This is write-only, append-only stream that always buffers the wrapped stream + /// into the chunks of the same size (except the possible shorter last chunk). + /// So unlike the it never writes to the wrapped stream + /// until it has full chunk or is closing. + /// + /// This is not supposed to bring performance benefits, but it allows to avoid nondeterministic + /// GZipStream output for the identical input. + /// + internal class GreedyBufferedStream : Stream + { + private readonly Stream _stream; + private readonly byte[] _buffer; + private int _position; + + public GreedyBufferedStream(Stream stream, int bufferSize) + { + _stream = stream; + _buffer = new byte[bufferSize]; + } + + public override void Flush() + { + _stream.Write(_buffer, 0, _position); + _position = 0; + } + + public override int Read(byte[] buffer, int offset, int count) => throw UnsupportedException; + + public override long Seek(long offset, SeekOrigin origin) => throw UnsupportedException; + + public override void SetLength(long value) => throw UnsupportedException; + + public override void Write(byte[] buffer, int offset, int count) + { + // Appends input to the buffer until it is full - then flushes it to the wrapped stream. + // Repeat above until all input is processed. + + int srcOffset = offset; + do + { + int currentCount = Math.Min(count, _buffer.Length - _position); + Buffer.BlockCopy(buffer, srcOffset, _buffer, _position, currentCount); + _position += currentCount; + count -= currentCount; + srcOffset += currentCount; + + if (_position == _buffer.Length) + { + Flush(); + } + } while (count > 0); + } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => _stream.CanWrite; + public override long Length => _stream.Length + _position; + + public override long Position + { + get => _stream.Position + _position; + set => throw UnsupportedException; + } + + public override void Close() + { + Flush(); + _stream.Close(); + base.Close(); + } + + private Exception UnsupportedException => new NotSupportedException("GreedyBufferedStream is write-only, append-only"); + } +} diff --git a/src/Build/Logging/BinaryLogger/IBuildEventArgsReaderNotifications.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs similarity index 89% rename from src/Build/Logging/BinaryLogger/IBuildEventArgsReaderNotifications.cs rename to src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 415bd7c71fd..41ec404904e 100644 --- a/src/Build/Logging/BinaryLogger/IBuildEventArgsReaderNotifications.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -6,7 +6,7 @@ namespace Microsoft.Build.Logging /// /// An interface for notifications from BuildEventArgsReader /// - public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader + public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IEmbeddedContentSource { /* For future use */ } diff --git a/src/Build/Logging/BinaryLogger/IBuildEventStringsReader.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs similarity index 100% rename from src/Build/Logging/BinaryLogger/IBuildEventStringsReader.cs rename to src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs new file mode 100644 index 00000000000..e0080148b1a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging +{ + public interface IEmbeddedContentSource + { + int FileFormatVersion { get; } + event Action EmbeddedContentRead; + } +} diff --git a/src/Build/Logging/BinaryLogger/StringReadEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/StringReadEventArgs.cs similarity index 100% rename from src/Build/Logging/BinaryLogger/StringReadEventArgs.cs rename to src/Build/Logging/BinaryLogger/Postprocessing/StringReadEventArgs.cs diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index 39a953f2948..62a4c56e0cf 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -159,9 +159,15 @@ - - - + + + + + + + + + diff --git a/src/Framework/Traits.cs b/src/Framework/Traits.cs index 9be74ea1bc8..864119f99f2 100644 --- a/src/Framework/Traits.cs +++ b/src/Framework/Traits.cs @@ -120,6 +120,12 @@ public Traits() /// public readonly int LogPropertyTracking = ParseIntFromEnvironmentVariableOrDefault("MsBuildLogPropertyTracking", 0); // Default to logging nothing via the property tracker. + /// + /// Turn on greedy buffering stream decorator for binlog writer. + /// This will ensure that 2 identical binlog contents will result into identical binlog files (as writing different chunks to GZipStream can lead to different result). + /// + public readonly bool DeterministicBinlogStreamBuffering = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MSBUILDDETERMNISTICBINLOG")); + /// /// When evaluating items, this is the minimum number of items on the running list to use a dictionary-based remove optimization. /// From 282de14076d8022415f8203deaad23b53c883601 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 15 Sep 2023 15:53:09 +0200 Subject: [PATCH 2/8] Added support for embedded files; fixing, cleaning --- .../BinaryLogReplayEventSource.cs | 21 +- .../Logging/BinaryLogger/BinaryLogger.cs | 82 +++---- .../BinaryLogger/BuildEventArgsReader.cs | 75 ++++++- .../BinaryLogger/BuildEventArgsWriter.cs | 13 +- .../Postprocessing/ArchiveFile.cs | 51 ++--- .../Postprocessing/ArchiveFileEventArgs.cs | 47 ++++ .../ArchiveFileEventArgsExtensions.cs | 23 ++ .../Postprocessing/CleanupScope.cs | 15 ++ .../EmbeddedContentEventArgs.cs | 8 +- .../IBuildEventArgsReaderNotifications.cs | 3 +- .../IBuildEventStringsReader.cs | 6 - .../Postprocessing/IBuildFileReader.cs | 34 +++ .../Postprocessing/IEmbeddedContentSource.cs | 15 +- .../BinaryLogger/Postprocessing/SubStream.cs | 56 +++++ .../BinaryLogger/ProjectImportsCollector.cs | 211 ++++++++++++------ src/Build/Microsoft.Build.csproj | 5 + 16 files changed, 511 insertions(+), 154 deletions(-) create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index b21169ed135..b6128c390d2 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -16,7 +16,7 @@ namespace Microsoft.Build.Logging /// by implementing IEventSource and raising corresponding events. /// /// The class is public so that we can call it from MSBuild.exe when replaying a log file. - public sealed class BinaryLogReplayEventSource : EventArgsDispatcher + public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IEmbeddedContentSource { /// Touches the static constructor /// to ensure it initializes @@ -125,10 +125,29 @@ public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken /// A indicating the replay should stop as soon as possible. public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken) { + _fileFormatVersionRead?.Invoke(reader.FileFormatVersion); + reader.EmbeddedContentRead += _embeddedContentRead; + while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) { Dispatch(instance); } } + + private Action? _fileFormatVersionRead; + event Action ILogVersionInfo.FileFormatVersionRead + { + add => _fileFormatVersionRead += value; + remove => _fileFormatVersionRead -= value; + } + private Action? _embeddedContentRead; + /// + event Action? IEmbeddedContentSource.EmbeddedContentRead + { + // Explicitly implemented event has to declare explicit add/remove accessors + // https://stackoverflow.com/a/2268472/2308106 + add => _embeddedContentRead += value; + remove => _embeddedContentRead -= value; + } } } diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index a95480428dd..9a43d3e30de 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -93,7 +93,12 @@ public enum ProjectImportsCollectionMode /// /// Create an external .ProjectImports.zip archive for the project files. /// - ZipFile + ZipFile, + + /// + /// Don't collect any files from build events, but instead replay them from the given event source (if that one supports it). + /// + Replay, } /// @@ -115,15 +120,10 @@ public enum ProjectImportsCollectionMode /// public string Parameters { get; set; } - /// - /// Initializes the logger by subscribing to events of the specified event source. - /// - public void Initialize(IEventSource eventSource) => Initialize(eventSource, null); - /// /// Initializes the logger by subscribing to events of the specified event source and embedded content source. /// - public void Initialize(IEventSource eventSource, IEmbeddedContentSource embeddedFilesSource) + public void Initialize(IEventSource eventSource) { _initialTargetOutputLogging = Environment.GetEnvironmentVariable("MSBUILDTARGETOUTPUTLOGGING"); _initialLogImports = Traits.Instance.EscapeHatches.LogProjectImports; @@ -136,14 +136,9 @@ public void Initialize(IEventSource eventSource, IEmbeddedContentSource embedded Traits.Instance.EscapeHatches.LogProjectImports = true; bool logPropertiesAndItemsAfterEvaluation = Traits.Instance.EscapeHatches.LogPropertiesAndItemsAfterEvaluation ?? true; - ProcessParameters(); - - if (embeddedFilesSource != null) - { - CollectProjectImports = ProjectImportsCollectionMode.None; - embeddedFilesSource.EmbeddedContentRead += args => - eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream, args.Length); - } + bool replayInitialInfo; + ILogVersionInfo versionInfo = null; + ProcessParameters(out replayInitialInfo); try { @@ -165,7 +160,7 @@ public void Initialize(IEventSource eventSource, IEmbeddedContentSource embedded stream = new FileStream(FilePath, FileMode.Create); - if (CollectProjectImports != ProjectImportsCollectionMode.None) + if (CollectProjectImports != ProjectImportsCollectionMode.None && CollectProjectImports != ProjectImportsCollectionMode.Replay) { projectImportsCollector = new ProjectImportsCollector(FilePath, CollectProjectImports == ProjectImportsCollectionMode.ZipFile); } @@ -179,6 +174,20 @@ public void Initialize(IEventSource eventSource, IEmbeddedContentSource embedded { eventSource4.IncludeEvaluationPropertiesAndItems(); } + + if (eventSource is IEmbeddedContentSource embeddedFilesSource) + { + if (CollectProjectImports == ProjectImportsCollectionMode.Replay) + { + embeddedFilesSource.EmbeddedContentRead += args => + eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream, args.Length); + } + + if (replayInitialInfo) + { + versionInfo = embeddedFilesSource; + } + } } catch (Exception e) { @@ -204,14 +213,14 @@ public void Initialize(IEventSource eventSource, IEmbeddedContentSource embedded eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile; } - if (embeddedFilesSource == null) + if (versionInfo == null) { binaryWriter.Write(FileFormatVersion); LogInitialInfo(); } else { - binaryWriter.Write(embeddedFilesSource.FileFormatVersion); + versionInfo.FileFormatVersionRead += version => binaryWriter.Write(version); } eventSource.AnyEventRaised += EventSource_AnyEventRaised; @@ -251,32 +260,18 @@ public void Shutdown() Traits.Instance.EscapeHatches.LogProjectImports = _initialLogImports; + if (projectImportsCollector != null) { projectImportsCollector.Close(); if (CollectProjectImports == ProjectImportsCollectionMode.Embed) { - var archiveFilePath = projectImportsCollector.ArchiveFilePath; + projectImportsCollector.ProcessResult( + streamToEmbed => eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, streamToEmbed), + LogMessage); - // It is possible that the archive couldn't be created for some reason. - // Only embed it if it actually exists. - if (FileSystems.Default.FileExists(archiveFilePath)) - { - using (FileStream fileStream = File.OpenRead(archiveFilePath)) - { - if (fileStream.Length > int.MaxValue) - { - LogMessage("Imported files archive exceeded 2GB limit and it's not embedded."); - } - else - { - eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, fileStream); - } - } - - File.Delete(archiveFilePath); - } + projectImportsCollector.DeleteArchive(); } projectImportsCollector = null; @@ -325,7 +320,7 @@ private void CollectImports(BuildEventArgs e) { projectImportsCollector.AddFile(projectArgs.ProjectFile); } - else if (e is MetaprojectGeneratedEventArgs metaprojectArgs) + else if (e is MetaprojectGeneratedEventArgs { metaprojectXml: { } } metaprojectArgs) { projectImportsCollector.AddFileFromMemory(metaprojectArgs.ProjectFile, metaprojectArgs.metaprojectXml); } @@ -340,13 +335,14 @@ private void CollectImports(BuildEventArgs e) /// /// /// - private void ProcessParameters() + private void ProcessParameters(out bool replayInitialInfo) { if (Parameters == null) { throw new LoggerException(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("InvalidBinaryLoggerParameters", "")); } + replayInitialInfo = false; var parameters = Parameters.Split(MSBuildConstants.SemicolonChar, StringSplitOptions.RemoveEmptyEntries); foreach (var parameter in parameters) { @@ -362,6 +358,14 @@ private void ProcessParameters() { CollectProjectImports = ProjectImportsCollectionMode.ZipFile; } + else if (string.Equals(parameter, "ProjectImports=Replay", StringComparison.OrdinalIgnoreCase)) + { + CollectProjectImports = ProjectImportsCollectionMode.Replay; + } + else if (string.Equals(parameter, "ReplayInitialInfo", StringComparison.OrdinalIgnoreCase)) + { + replayInitialInfo = true; + } else if (parameter.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) { FilePath = parameter; diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 4af1ecd28b8..fb7dacd422a 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -5,6 +5,8 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.IO.Compression; +using System.Linq; using System.Reflection; using System.Text; using Microsoft.Build.BackEnd; @@ -98,7 +100,10 @@ public void Dispose() /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. /// If no subscriber is attached, the data is skipped. /// - public event Action? EmbeddedContentRead; + internal event Action? EmbeddedContentRead; + + /// + public event Action? ArchiveFileEncountered; /// /// Reads the next log record from the . @@ -230,16 +235,72 @@ private static bool IsAuxiliaryRecord(BinaryLogRecordKind recordKind) private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) { int length = ReadInt32(); - if (EmbeddedContentRead != null) + + if (ArchiveFileEncountered != null) { - long preEventPosition = binaryReader.BaseStream.CanSeek ? binaryReader.BaseStream.Position : 0; - EmbeddedContentRead(new EmbeddedContentEventArgs(recordKind.ToEmbeddedContentKind(), binaryReader.BaseStream, length)); - long postEventPosition = binaryReader.BaseStream.CanSeek ? binaryReader.BaseStream.Position : length; - if (postEventPosition - preEventPosition != length) + // We could create ZipArchive over the target stream, and write to that directly, + // however, binlog format needs to know stream size upfront - which is unknown, + // so we would need to write the size after - and that would require the target stream to be seekable (which it's not) + ProjectImportsCollector? projectImportsCollector = null; + + if (EmbeddedContentRead != null) + { + projectImportsCollector = + new ProjectImportsCollector(Path.GetRandomFileName(), false, runOnBackground: false); + } + + Stream embeddedStream = new SubStream(binaryReader.BaseStream, length); + + // We are intentionally not grace handling corrupt embedded stream + + using var zipArchive = new ZipArchive(embeddedStream, ZipArchiveMode.Read); + + foreach (var entry in zipArchive.Entries/*.OrderBy(e => e.LastWriteTime)*/) + { + var file = ArchiveFile.From(entry); + ArchiveFileEventArgs archiveFileEventArgs = new(file); + // ArchiveFileEventArgs is not IDisposable as we do not want to clutter exposed API + using var cleanupScope = new CleanupScope(archiveFileEventArgs.Dispose); + ArchiveFileEncountered(archiveFileEventArgs); + + if (projectImportsCollector != null) + { + var resultFile = archiveFileEventArgs.ObtainArchiveFile(); + + if (resultFile.CanUseReader) + { + projectImportsCollector.AddFileFromMemory( + resultFile.FullPath, + resultFile.GetContentReader().BaseStream, + makePathAbsolute: false, + entryCreationStamp: entry.LastWriteTime); + } + else + { + projectImportsCollector.AddFileFromMemory( + resultFile.FullPath, + resultFile.GetContent(), + encoding: resultFile.Encoding, + makePathAbsolute: false, + entryCreationStamp: entry.LastWriteTime); + } + } + } + + if (EmbeddedContentRead != null) { - throw new InvalidDataException($"The {nameof(EmbeddedContentRead)} event handler must read exactly {length} bytes from the stream."); + projectImportsCollector!.ProcessResult( + streamToEmbed => EmbeddedContentRead(new EmbeddedContentEventArgs(EmbeddedContentKind.ProjectImportArchive, streamToEmbed)), + error => throw new InvalidDataException(error)); + projectImportsCollector.DeleteArchive(); } } + else if (EmbeddedContentRead != null) + { + EmbeddedContentRead(new EmbeddedContentEventArgs( + recordKind.ToEmbeddedContentKind(), + new SubStream(binaryReader.BaseStream, length))); + } else { if (binaryReader.BaseStream.CanSeek) diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs index 3d1e9258e75..ba6956cee42 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs @@ -211,13 +211,7 @@ private void WriteCore(BuildEventArgs e) } } - public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int length) - => WriteBlobImpl(kind, stream, length); - - public void WriteBlob(BinaryLogRecordKind kind, Stream stream) - => WriteBlobImpl(kind, stream, null); - - private void WriteBlobImpl(BinaryLogRecordKind kind, Stream stream, int? length) + public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int? length = null) { if (stream.CanSeek && stream.Length > int.MaxValue) { @@ -1092,9 +1086,10 @@ private void Write(byte[] bytes) private void Write(Stream stream, int? length) { + Stream destinationStream = binaryWriter.BaseStream; if (length == null) { - stream.CopyTo(binaryWriter.BaseStream); + stream.CopyTo(destinationStream); return; } @@ -1110,7 +1105,7 @@ private void Write(Stream stream, int? length) length > 0 && (bytesRead = stream.Read(buffer, 0, Math.Min(buffer.Length, length.Value))) != 0) { - binaryWriter.BaseStream.Write(buffer, 0, bytesRead); + destinationStream.Write(buffer, 0, bytesRead); length -= bytesRead; } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index 59868eb43dd..5f3f75ab761 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -4,29 +4,46 @@ using System; using System.IO; using System.IO.Compression; +using System.Text; namespace Microsoft.Build.Logging { - public class ArchiveFile : IDisposable + public class ArchiveFile { - public ArchiveFile(string fullPath, StreamReader contentReader) + public ArchiveFile(string fullPath, Stream contentStream) { FullPath = fullPath; - _contentReader = contentReader; + // We need to specify encoding without preamble - as then StreamReader will + // automatically adjust the encoding to match the preamble (if present). + _contentReader = new StreamReader(contentStream, new System.Text.UTF8Encoding(false)); + } + + public ArchiveFile(string fullPath, string content, Encoding? contentEncoding = null) + { + FullPath = fullPath; + _content = content; + _stringAcquired = true; + _contentReader = StreamReader.Null; + _stringEncoding = contentEncoding ?? Encoding.UTF8; } public static ArchiveFile From(ZipArchiveEntry entry) { - return new ArchiveFile(CalculateArchivePath(entry.FullName), new StreamReader(entry.Open())); + return new ArchiveFile(entry.FullName, entry.Open()); } public string FullPath { get; } + public Encoding Encoding => _stringEncoding ?? _contentReader.CurrentEncoding; + + public bool CanUseReader => !_stringAcquired; + public bool CanUseString => !_streamAcquired; + public StreamReader GetContentReader() { if (_stringAcquired) { - throw new InvalidOperationException("Content already acquired as string via GetContent."); + throw new InvalidOperationException("Content already acquired as string via GetContent or initialized as string only."); } _streamAcquired = true; @@ -53,26 +70,10 @@ public string GetContent() private bool _stringAcquired; private readonly StreamReader _contentReader; private string? _content; + private readonly Encoding? _stringEncoding; - public static string CalculateArchivePath(string filePath) - { - string archivePath = filePath; - - if (filePath.Contains(":") || (!filePath.StartsWith("\\") && !filePath.StartsWith("/"))) - { - archivePath = archivePath.Replace(":", ""); - archivePath = archivePath.Replace("/", "\\"); - archivePath = archivePath.Replace("\\\\", "\\"); - } - else - { - archivePath = archivePath.Replace("\\", "/"); - archivePath = archivePath.Replace("//", "/"); - } - - return archivePath; - } - - public void Dispose() => _contentReader.Dispose(); + // Intentionally not exposing this publicly (e.g. as IDisposable implementation) + // as we don't want to user to be bothered with ownership and disposing concerns. + internal void Dispose() => _contentReader.Dispose(); } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs new file mode 100644 index 00000000000..7045f261d86 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging; + +public class ArchiveFileEventArgs : EventArgs +{ + private ArchiveFile _archiveFile; + private bool _resultSet; + private Action _disposeAction; + + public ArchiveFileEventArgs(ArchiveFile archiveFile) => + (_archiveFile, _resultSet, _disposeAction) = (archiveFile, true, archiveFile.Dispose); + + public ArchiveFile ObtainArchiveFile() + { + if (!_resultSet) + { + throw new InvalidOperationException( + "ArchiveFile was obtained, but the final edited version was not set."); + } + + _resultSet = false; + return _archiveFile; + } + + public void SetResult(string resultPath, Stream resultStream) + { + _archiveFile = new ArchiveFile(resultPath, resultStream); + _disposeAction += _archiveFile.Dispose; + _resultSet = true; + } + + public void SetResult(string resultPath, string resultContent) + { + _archiveFile = new ArchiveFile(resultPath, resultContent, _archiveFile.Encoding); + _disposeAction += _archiveFile.Dispose; + _resultSet = true; + } + + // Intentionally not exposing this publicly (e.g. as IDisposable implementation) + // as we don't want to user to be bothered with ownership and disposing concerns. + internal void Dispose() => _disposeAction(); +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs new file mode 100644 index 00000000000..818cffaa91a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +public static class ArchiveFileEventArgsExtensions +{ + public static Action ToArchiveFileHandler(this Action stringHandler) + { + return args => + { + var archiveFile = args.ObtainArchiveFile(); + var pathArgs = new StringReadEventArgs(archiveFile.FullPath); + stringHandler(pathArgs); + var contentArgs = new StringReadEventArgs(archiveFile.GetContent()); + stringHandler(contentArgs); + + args.SetResult(pathArgs.StringToBeUsed, contentArgs.StringToBeUsed); + }; + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs b/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs new file mode 100644 index 00000000000..bef26ff4d13 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +internal class CleanupScope : IDisposable +{ + private readonly Action _disposeAction; + + public CleanupScope(Action disposeAction) => _disposeAction = disposeAction; + + public void Dispose() => _disposeAction(); +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs index b5d6e985783..ce372c888d3 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -15,8 +15,14 @@ public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentS Length = length; } + public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream) + { + ContentKind = contentKind; + ContentStream = contentStream; + } + public EmbeddedContentKind ContentKind { get; } public Stream ContentStream { get; } - public int Length { get; } + public int? Length { get; } } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 41ec404904e..5f49119570f 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -6,7 +6,8 @@ namespace Microsoft.Build.Logging /// /// An interface for notifications from BuildEventArgsReader /// - public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IEmbeddedContentSource + // todo: IEmbeddedContentSource should not be here ideally + public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IBuildFileReader { /* For future use */ } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs index e9e7651ee78..bf3d54f8ff8 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs @@ -16,11 +16,5 @@ public interface IBuildEventStringsReader /// The passed event arg can be reused and should not be stored. /// public event Action? StringReadDone; - - /// - /// An event that allows the caller to be notified when a string is encountered in the binary log. - /// BinaryReader passed in ctor is at the beginning of the string at this point. - /// - public event Action? StringEncountered; } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs new file mode 100644 index 00000000000..9910ee1a06a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +public interface IBuildFileReader +{ + /// + /// An event that allows the caller to be notified when an embedded file is encountered in the binary log. + /// Subscribing to this event obligates the subscriber to read the file content and set the resulting content + /// via or . + /// When subscriber is OK with greedy reading entire content of the file, it can simplify subscribing to this event, + /// by using handler with same signature as handler for and wrapping it via + /// extension. + /// + /// + /// + /// private void OnStringReadDone(StringReadEventArgs e) + /// { + /// e.StringToBeUsed = e.StringToBeUsed.Replace("foo", "bar"); + /// } + /// + /// private void SubscribeToEvents() + /// { + /// reader.StringReadDone += OnStringReadDone; + /// reader.ArchiveFileEncountered += ((Action<StringReadEventArgs>)OnStringReadDone).ToArchiveFileHandler(); + /// } + /// + /// + /// + public event Action? ArchiveFileEncountered; +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs index e0080148b1a..eb9262939fa 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs @@ -5,9 +5,20 @@ namespace Microsoft.Build.Logging { - public interface IEmbeddedContentSource + + internal interface ILogVersionInfo + { + event Action FileFormatVersionRead; + } + + internal interface IEmbeddedContentSource : ILogVersionInfo { - int FileFormatVersion { get; } + + /// + /// Raised when the log reader encounters a project import archive (embedded content) in the stream. + /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. + /// If no subscriber is attached, the data is skipped. + /// event Action EmbeddedContentRead; } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs new file mode 100644 index 00000000000..277c9ac66c0 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Build.Logging +{ + /// + /// Bounded read-only, forward-only view over an underlying stream. + /// + internal class SubStream : Stream + { + // Do not Dispose/Close on Dispose/Close !! + private readonly Stream _stream; + private readonly long _length; + private long _position; + + public SubStream(Stream stream, long length) + { + _stream = stream; + _length = length; + + if (!stream.CanRead) + { + throw new InvalidOperationException("Stream must be readable."); + } + } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => _length; + + public override long Position { get => _position; set => throw new NotImplementedException(); } + + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) + { + count = Math.Min((int)Math.Max(Length - _position, 0), count); + int read = _stream.Read(buffer, offset, count); + _position += read; + return read; + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); + public override void SetLength(long value) => throw new NotImplementedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + } +} diff --git a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs index 27ededae8cc..24e86af2991 100644 --- a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs +++ b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs @@ -8,8 +8,7 @@ using System.Text; using System.Threading.Tasks; using Microsoft.Build.Shared; - -#nullable disable +using Microsoft.Build.Shared.FileSystem; namespace Microsoft.Build.Logging { @@ -21,10 +20,10 @@ namespace Microsoft.Build.Logging /// internal class ProjectImportsCollector { - private Stream _fileStream; - private ZipArchive _zipArchive; - - public string ArchiveFilePath { get; } + private Stream? _fileStream; + private ZipArchive? _zipArchive; + private readonly string _archiveFilePath; + private readonly bool _runOnBackground; /// /// Avoid visiting each file more than once. @@ -34,12 +33,16 @@ internal class ProjectImportsCollector // this will form a chain of file write tasks, running sequentially on a background thread private Task _currentTask = Task.CompletedTask; - public ProjectImportsCollector(string logFilePath, bool createFile, string sourcesArchiveExtension = ".ProjectImports.zip") + public ProjectImportsCollector( + string logFilePath, + bool createFile, + string sourcesArchiveExtension = ".ProjectImports.zip", + bool runOnBackground = true) { if (createFile) { // Archive file will be stored alongside the binlog - ArchiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); + _archiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); } else { @@ -50,7 +53,7 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc } // Archive file will be temporarily stored in MSBuild cache folder and deleted when no longer needed - ArchiveFilePath = Path.Combine( + _archiveFilePath = Path.Combine( cacheDirectory, Path.ChangeExtension( Path.GetFileName(logFilePath), @@ -59,7 +62,7 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc try { - _fileStream = new FileStream(ArchiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); + _fileStream = new FileStream(_archiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); _zipArchive = new ZipArchive(_fileStream, ZipArchiveMode.Create); } catch @@ -69,50 +72,72 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc _fileStream = null; _zipArchive = null; } + _runOnBackground = runOnBackground; + } + + public void AddFile(string? filePath) + { + AddFileHelper(filePath, AddFileCore); } - public void AddFile(string filePath) + public void AddFileFromMemory( + string? filePath, + string data, + Encoding? encoding = null, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) + { + AddFileHelper(filePath, path => + AddFileFromMemoryCore(path, data, encoding ?? Encoding.UTF8, makePathAbsolute, entryCreationStamp)); + } + + public void AddFileFromMemory( + string? filePath, + Stream data, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) + { + AddFileHelper(filePath, path => AddFileFromMemoryCore(path, data, makePathAbsolute, entryCreationStamp)); + } + + private void AddFileHelper( + string? filePath, + Action addFileWorker) { if (filePath != null && _fileStream != null) { + Action addFileAction = WrapWithExceptionSwallowing(() => addFileWorker(filePath)); + lock (_fileStream) { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => + if (_runOnBackground) + { + // enqueue the task to add a file and return quickly + // to avoid holding up the current thread + _currentTask = _currentTask.ContinueWith( + t => { addFileAction(); }, + TaskScheduler.Default); + } + else { - try - { - AddFileCore(filePath); - } - catch - { - } - }, TaskScheduler.Default); + addFileAction(); + } } } } - public void AddFileFromMemory(string filePath, string data) + private Action WrapWithExceptionSwallowing(Action action) { - if (filePath != null && data != null && _fileStream != null) + return () => { - lock (_fileStream) + try { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => - { - try - { - AddFileFromMemoryCore(filePath, data); - } - catch - { - } - }, TaskScheduler.Default); + action(); } - } + catch + { + } + }; } /// @@ -122,61 +147,94 @@ public void AddFileFromMemory(string filePath, string data) private void AddFileCore(string filePath) { // quick check to avoid repeated disk access for Exists etc. - if (_processedFiles.Contains(filePath)) + if (!ShouldAddFile(ref filePath, true, true)) { return; } - if (!File.Exists(filePath)) + using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); + AddFileData(filePath, content, null); + } + + /// + /// This method doesn't need locking/synchronization because it's only called + /// from a task that is chained linearly + /// + private void AddFileFromMemoryCore(string filePath, string data, Encoding encoding, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { - _processedFiles.Add(filePath); return; } - filePath = Path.GetFullPath(filePath); + AddFileData(filePath, data, encoding, entryCreationStamp); + } - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + private void AddFileFromMemoryCore(string filePath, Stream data, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { return; } - using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); - using Stream entryStream = OpenArchiveEntry(filePath); - content.CopyTo(entryStream); + AddFileData(filePath, data, entryCreationStamp); } - /// - /// This method doesn't need locking/synchronization because it's only called - /// from a task that is chained linearly - /// - private void AddFileFromMemoryCore(string filePath, string data) + private void AddFileData(string filePath, Stream data, DateTimeOffset? entryCreationStamp) + { + using Stream entryStream = OpenArchiveEntry(filePath, entryCreationStamp); + data.CopyTo(entryStream); + } + + private void AddFileData(string filePath, string data, Encoding encoding, DateTimeOffset? entryCreationStamp) + { + using Stream entryStream = OpenArchiveEntry(filePath, entryCreationStamp); + using MemoryStream memoryStream = new MemoryStream(); + // We need writer as encoding.GetBytes() isn't obliged to output preamble + // We cannot write directly to entryStream (preamble is written separately) as it's compressed differnetly, then writing the whole stream at once + using StreamWriter writer = new StreamWriter(memoryStream, encoding); + writer.Write(data); + writer.Flush(); + memoryStream.Position = 0; + memoryStream.CopyTo(entryStream); + } + + private bool ShouldAddFile(ref string filePath, bool checkFileExistence, bool makeAbsolute) { // quick check to avoid repeated disk access for Exists etc. if (_processedFiles.Contains(filePath)) { - return; + return false; } - filePath = Path.GetFullPath(filePath); - - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + if (checkFileExistence && !File.Exists(filePath)) { - return; + _processedFiles.Add(filePath); + return false; } - using (Stream entryStream = OpenArchiveEntry(filePath)) - using (var content = new MemoryStream(Encoding.UTF8.GetBytes(data))) + // Only make the path absolute if it's request. In the replay scenario, the file entries + // are read from zip archive - where ':' is stripped and path can then seem relative. + if (makeAbsolute) { - content.CopyTo(entryStream); + filePath = Path.GetFullPath(filePath); } + + // if the file is already included, don't include it again + return _processedFiles.Add(filePath); } - private Stream OpenArchiveEntry(string filePath) + private Stream OpenArchiveEntry(string filePath, DateTimeOffset? entryCreationStamp) { string archivePath = CalculateArchivePath(filePath); - var archiveEntry = _zipArchive.CreateEntry(archivePath); + var archiveEntry = _zipArchive!.CreateEntry(archivePath); + if (entryCreationStamp.HasValue) + { + archiveEntry.LastWriteTime = entryCreationStamp.Value; + } + return archiveEntry.Open(); } @@ -191,6 +249,27 @@ private static string CalculateArchivePath(string filePath) return archivePath; } + public void ProcessResult(Action consumeStream, Action onError) + { + Close(); + + // It is possible that the archive couldn't be created for some reason. + // Only embed it if it actually exists. + if (FileSystems.Default.FileExists(_archiveFilePath)) + { + using FileStream fileStream = File.OpenRead(_archiveFilePath); + + if (fileStream.Length > int.MaxValue) + { + onError("Imported files archive exceeded 2GB limit and it's not embedded."); + } + else + { + consumeStream(fileStream); + } + } + } + public void Close() { // wait for all pending file writes to complete @@ -208,5 +287,11 @@ public void Close() _fileStream = null; } } + + public void DeleteArchive() + { + Close(); + File.Delete(_archiveFilePath); + } } } diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index 62a4c56e0cf..b3e03043a4e 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -160,14 +160,19 @@ + + + + + From 19fd9137be6e82e9fb695d7cef6b85f4f075ed70 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Mon, 18 Sep 2023 14:26:55 +0200 Subject: [PATCH 3/8] Decrease scope where accidentally exposed --- .../BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs | 2 +- .../Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs | 2 +- .../Postprocessing/IBuildEventArgsReaderNotifications.cs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs index ce372c888d3..68969f2af4a 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -6,7 +6,7 @@ namespace Microsoft.Build.Logging { - public sealed class EmbeddedContentEventArgs : EventArgs + internal sealed class EmbeddedContentEventArgs : EventArgs { public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream, int length) { diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs index 7f694c91363..2fca5d7eaa3 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs @@ -9,7 +9,7 @@ namespace Microsoft.Build.Logging { - public enum EmbeddedContentKind + internal enum EmbeddedContentKind { Unknown = -1, ProjectImportArchive = 17, diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 5f49119570f..13bc343362a 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -6,7 +6,6 @@ namespace Microsoft.Build.Logging /// /// An interface for notifications from BuildEventArgsReader /// - // todo: IEmbeddedContentSource should not be here ideally public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IBuildFileReader { /* For future use */ From 6e03efed6c85f1d6fae5ad9b48c238603b282c96 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 19 Sep 2023 15:17:04 +0200 Subject: [PATCH 4/8] Sealing types --- src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs | 2 +- .../Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index 5f3f75ab761..6c6a9daf26c 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -8,7 +8,7 @@ namespace Microsoft.Build.Logging { - public class ArchiveFile + public sealed class ArchiveFile { public ArchiveFile(string fullPath, Stream contentStream) { diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs index 7045f261d86..45bc252a066 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -6,7 +6,7 @@ namespace Microsoft.Build.Logging; -public class ArchiveFileEventArgs : EventArgs +public sealed class ArchiveFileEventArgs : EventArgs { private ArchiveFile _archiveFile; private bool _resultSet; From 8a3bb35c15e0bc1069ed9fdf11180b6b6eba86b1 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 20 Sep 2023 16:50:12 +0200 Subject: [PATCH 5/8] Adjust visibility, add comments --- .../BinaryLogger/Postprocessing/ArchiveFile.cs | 14 +++++++++++++- .../Postprocessing/ArchiveFileEventArgs.cs | 7 +++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index 6c6a9daf26c..5e06a41362a 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -27,7 +27,7 @@ public ArchiveFile(string fullPath, string content, Encoding? contentEncoding = _stringEncoding = contentEncoding ?? Encoding.UTF8; } - public static ArchiveFile From(ZipArchiveEntry entry) + internal static ArchiveFile From(ZipArchiveEntry entry) { return new ArchiveFile(entry.FullName, entry.Open()); } @@ -39,6 +39,12 @@ public static ArchiveFile From(ZipArchiveEntry entry) public bool CanUseReader => !_stringAcquired; public bool CanUseString => !_streamAcquired; + /// + /// Fetches the file content as a stream reader (forward only). + /// This prevents the content to be read as string. + /// + /// + /// public StreamReader GetContentReader() { if (_stringAcquired) @@ -50,6 +56,12 @@ public StreamReader GetContentReader() return _contentReader; } + /// + /// Fetches the file content as a string. + /// This prevents the content to be fetched via StreamReader. + /// + /// + /// public string GetContent() { if (_streamAcquired) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs index 45bc252a066..120362bcf55 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -15,6 +15,13 @@ public sealed class ArchiveFileEventArgs : EventArgs public ArchiveFileEventArgs(ArchiveFile archiveFile) => (_archiveFile, _resultSet, _disposeAction) = (archiveFile, true, archiveFile.Dispose); + /// + /// Acquires the instance. This method can only be called once and + /// or must be called afterwards + /// (this is because the embedded files are stored as forward only stream - reading them prevents re-reads). + /// + /// + /// public ArchiveFile ObtainArchiveFile() { if (!_resultSet) From 09f32ed4fc30fd893bbc9747fafec5e5a84c1f6e Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 21 Sep 2023 12:07:47 +0200 Subject: [PATCH 6/8] Cache encoding --- .../Logging/BinaryLogger/Postprocessing/ArchiveFile.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index 5e06a41362a..88e80cd9d59 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -10,12 +10,15 @@ namespace Microsoft.Build.Logging { public sealed class ArchiveFile { + // We need to specify encoding without preamble - as then StreamReader will + // automatically adjust the encoding to match the preamble (if present). + // It will as well change to other encoding if detected. + private static readonly Encoding s_utf8WithoutPreamble = new UTF8Encoding(false); + public ArchiveFile(string fullPath, Stream contentStream) { FullPath = fullPath; - // We need to specify encoding without preamble - as then StreamReader will - // automatically adjust the encoding to match the preamble (if present). - _contentReader = new StreamReader(contentStream, new System.Text.UTF8Encoding(false)); + _contentReader = new StreamReader(contentStream, s_utf8WithoutPreamble); } public ArchiveFile(string fullPath, string content, Encoding? contentEncoding = null) From 7cdd0e09292fb475e2b5f7560d9cdbceead9eded Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Mon, 2 Oct 2023 14:12:58 +0200 Subject: [PATCH 7/8] Fix code after merge --- .../Logging/BinaryLogger/BuildEventArgsReader.cs | 11 ++--------- src/Build/Microsoft.Build.csproj | 3 --- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index f2c6365cb6f..dedc7c8a1ab 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -87,12 +87,6 @@ public void Dispose() /// public event Action? StringReadDone; - /// - /// An event that allows the caller to be notified when a string is encountered in the binary log. - /// BinaryReader passed in ctor is at the beginning of the string at this point. - /// - public event Action? StringEncountered; - public int FileFormatVersion => fileFormatVersion; /// @@ -512,7 +506,7 @@ private BuildEventArgs ReadProjectEvaluationFinishedEventArgs() if (fileFormatVersion >= 12) { - if (fileFormatVersion < 17) + if (fileFormatVersion < 18) { // Throw away, but need to advance past it ReadBoolean(); @@ -568,7 +562,7 @@ private BuildEventArgs ReadProjectStartedEventArgs() if (fileFormatVersion > 6) { - if (fileFormatVersion < 17) + if (fileFormatVersion < 18) { // Throw away, but need to advance past it ReadBoolean(); @@ -1371,7 +1365,6 @@ private ITaskItem ReadTaskItem() private string ReadString() { - this.StringEncountered?.Invoke(); string text = binaryReader.ReadString(); if (this.StringReadDone != null) { diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index 12387bcce58..43cfb1b3f80 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -179,9 +179,6 @@ - - - From e970a9921617390bcac107eb9c354551f569702c Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 4 Oct 2023 10:28:27 +0200 Subject: [PATCH 8/8] Fix after merge --- src/Build/CompatibilitySuppressions.xml | 172 +++++++++++++++++- .../Logging/BinaryLogger/BinaryLogger.cs | 4 +- 2 files changed, 173 insertions(+), 3 deletions(-) diff --git a/src/Build/CompatibilitySuppressions.xml b/src/Build/CompatibilitySuppressions.xml index 05317adadab..23427182e98 100644 --- a/src/Build/CompatibilitySuppressions.xml +++ b/src/Build/CompatibilitySuppressions.xml @@ -1,4 +1,174 @@  - + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + \ No newline at end of file diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index c01095f73b9..94b28d10eb2 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -62,10 +62,10 @@ public sealed class BinaryLogger : ILogger // version 16: // - AssemblyLoadBuildEventArgs // version 17: + // - Added extended data for types implementing IExtendedBuildEventArgs + // version 18: // - Making ProjectStartedEventArgs, ProjectEvaluationFinishedEventArgs, AssemblyLoadBuildEventArgs equal // between de/serialization roundtrips. - // version 18: - // - Added extended data for types implementing IExtendedBuildEventArgs internal const int FileFormatVersion = 18; private Stream stream;