Incremental mibc merge (#87336)

Before this change, the merged mibc file would inherit the timestamp of the input (which are often days in the past, from the optimization data nuget package). This caused merging to happen on every incremental build, as dotnet-pgo.dll is always newer than the output file. The timestamp was inherited to fix a different incremental build issue, where updating to a newer optimization data package wouldn't make the input timestamp new enough to trigger a re-merge. See #56397 for context. This fixes both issues by caching the input modification timestamps, so that the merged output file will get a new timestamp when it is created, but will be considered out of date whenever the input timestamps change. This reduces incremental build times on my machine by ~3s. Contributes to #47022 Also deletes the --inherit-timestamp logic.
dotnet · Jun 9, 2023 · 454057d · 454057d
1 parent fa4f15e
commit 454057d
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 15 deletions.
diff --git a/eng/restore/optimizationData.targets b/eng/restore/optimizationData.targets
@@ -30,8 +30,9 @@
         <PackagePath>$(NuGetPackageRoot)%(MIBCPackage.Identity)/%(MIBCPackage.Version)</PackagePath>
       </MIBCPackage>
       <_optimizationMibcFile Include="%(MIBCPackage.PackagePath)/**/*.mibc" SubdirectoryName="$(TargetOS)/$(TargetArchitecture)" />
+      <_optimizationMibcDestinationFile Include="@(_optimizationMibcFile->'$(MibcOptimizationDataDir)%(SubdirectoryName)/%(RecursiveDir)%(Filename)%(Extension)')" />
       <ExcessFilesCurrentlyPresent Include="$(MibcOptimizationDataDir)/**" 
-                                   Exclude="@(_optimizationMibcFile->'$(MibcOptimizationDataDir)%(SubdirectoryName)/%(RecursiveDir)%(Filename)%(Extension)')"/>
+                                   Exclude="@(_optimizationMibcDestinationFile)"/>
     </ItemGroup>
 
     <Error Condition="'@(_optimizationMibcFile)' == ''" Text="Failed to restore Mibc optimization data" />
@@ -41,7 +42,7 @@
 
     <!-- Copy the correct mibc files into place -->
     <Copy SourceFiles="@(_optimizationMibcFile)"
-          DestinationFiles="@(_optimizationMibcFile->'$(MibcOptimizationDataDir)%(SubdirectoryName)/%(RecursiveDir)%(Filename)%(Extension)')"
+          DestinationFiles="@(_optimizationMibcDestinationFile)"
           SkipUnchangedFiles="true"
           UseHardlinksIfPossible="true" />
 

diff --git a/src/coreclr/crossgen-corelib.proj b/src/coreclr/crossgen-corelib.proj
@@ -27,6 +27,8 @@
 
     <BuildPerfMap>false</BuildPerfMap>
     <BuildPerfMap Condition="$(BuildDll) and '$(TargetOS)' == 'linux'">true</BuildPerfMap>
+
+    <_MergeMibcFilesCacheFile>$(MibcOptimizationDataDir)/$(TargetOS)/$(TargetArchitecture)/merge_mibc_files.cache</_MergeMibcFilesCacheFile>
   </PropertyGroup>
 
   <ItemGroup>
@@ -43,16 +45,41 @@
     <MergedMibcPath>$([MSBuild]::NormalizePath('$(BinDir)', 'StandardOptimizationData.mibc'))</MergedMibcPath>
   </PropertyGroup>
 
+  <!-- Creates a hash file that changes whenever the input mibc file modification times change.
+       This will trigger a re-merge when updating to a new version of the optimization data,
+       even if the new timestamps are still days in the past. -->
+  <Target Name="GenerateMergeMibcFilesInputCache">
+    <ItemGroup>
+      <_MergeMibcFilesInputCache Include="@(OptimizationMibcFiles)" />
+      <_MergeMibcFilesInputCache Include="@(DotNetPgo)" />
+      <_MergeMibcFilesInputCache Include="@(OptimizationMibcFiles->'%(ModifiedTime)')" />
+    </ItemGroup>
+
+    <Hash ItemsToHash="@(_MergeMibcFilesInputCache)">
+      <Output TaskParameter="HashResult" PropertyName="_MergeMibcFilesInputCacheHash" />
+    </Hash>
+
+    <WriteLinesToFile
+      Lines="$(_MergeMibcFilesInputCacheHash)"
+      File="$(_MergeMibcFilesCacheFile)"
+      Overwrite="true"
+      WriteOnlyWhenDifferent="true" />
+
+    <ItemGroup>
+      <FileWrites Include="$(_MergeMibcFilesCacheFile)" />
+    </ItemGroup>
+  </Target>
+
   <Target Name="MergeMibcFiles"
-          Inputs="@(OptimizationMibcFiles);@(DotNetPgo)"
+          DependsOnTargets="GenerateMergeMibcFilesInputCache"
+          Inputs="$(_MergeMibcFilesCacheFile);@(OptimizationMibcFiles);@(DotNetPgo)"
           Condition="'@(DotNetPgo)' != ''"
           Outputs="$(MergedMibcPath)">
 
     <PropertyGroup>
       <DotNetPgoCmd>$(DotNetCli) @(DotNetPgo) merge</DotNetPgoCmd>
       <DotNetPgoCmd>$(DotNetPgoCmd) -o:$(MergedMibcPath)</DotNetPgoCmd>
       <DotNetPgoCmd>$(DotNetPgoCmd) @(OptimizationMibcFiles->'-i:%(Identity)', ' ')</DotNetPgoCmd>
-      <DotNetPgoCmd>$(DotNetPgoCmd) --inherit-timestamp</DotNetPgoCmd> <!-- For incremental builds, otherwise timestamp is too far in the future -->
       <DotNetPgoCmd>$(DotNetPgoCmd) --compressed false</DotNetPgoCmd> <!-- Signing service doesn't handle compressed mibc signing correctly. -->
     </PropertyGroup>
 

diff --git a/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs b/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs
@@ -51,8 +51,6 @@ internal sealed class PgoRootCommand : RootCommand
             new("--dump-worst-overlap-graphs", () => -1, "Number of graphs to dump to .dot format in dump-worst-overlap-graphs-to directory");
         public Option<string> DumpWorstOverlapGraphsTo { get; } =
             new("--dump-worst-overlap-graphs-to", "Number of graphs to dump to .dot format in dump-worst-overlap-graphs-to directory");
-        public Option<bool> InheritTimestamp { get; } =
-            new("--inherit-timestamp", "If specified, set the output's timestamp to the max timestamp of the input files");
         public Option<bool> AutomaticReferences { get; } =
             new("--automatic-references", () => true, "Attempt to find references by using paths embedded in the trace file. Defaults to true");
         public Option<AssemblyName[]> IncludedAssemblies { get; } =
@@ -193,7 +191,6 @@ public PgoRootCommand(string[] args) : base(".NET PGO Tool")
                 InputFilesToMerge,
                 OutputFilePath,
                 IncludedAssemblies,
-                InheritTimestamp,
                 _verbosity,
                 Compressed
             };

diff --git a/src/coreclr/tools/dotnet-pgo/Program.cs b/src/coreclr/tools/dotnet-pgo/Program.cs
@@ -438,14 +438,7 @@ private int InnerMergeMain()
 
                 MibcConfig mergedConfig = ParseMibcConfigsAndMerge(tsc, mibcReaders);
                 var outputFileInfo = new FileInfo(outputPath);
-                int result = MibcEmitter.GenerateMibcFile(mergedConfig, tsc, outputFileInfo, mergedProfileData.Values, _command.ValidateOutputFile, !Get(_command.Compressed));
-                if (result == 0 && Get(_command.InheritTimestamp))
-                {
-                    outputFileInfo.CreationTimeUtc = paths.Max(f => new FileInfo(f).CreationTimeUtc);
-                    outputFileInfo.LastWriteTimeUtc = paths.Max(f => new FileInfo(f).LastWriteTimeUtc);
-                }
-
-                return result;
+                return MibcEmitter.GenerateMibcFile(mergedConfig, tsc, outputFileInfo, mergedProfileData.Values, _command.ValidateOutputFile, !Get(_command.Compressed));
             }
             finally
             {