Use hard link if possible when linking out files for upload (#42)

microsoft · Jan 23, 2024 · 8da7ac2 · 8da7ac2
1 parent 985d97f
commit 8da7ac2
Showing 1 changed file with 15 additions and 4 deletions.
diff --git a/src/AzurePipelines/PipelineCachingCacheClient.cs b/src/AzurePipelines/PipelineCachingCacheClient.cs
@@ -239,7 +239,12 @@ protected override async Task AddNodeAsync(
                     .Select(kvp => new ContentHashWithPath(kvp.Key, new AbsolutePath(kvp.Value)))
                     .ToList();
 
-                Dictionary<string, PlaceFileResult> placeResults = await TryPlaceFilesFromCacheAsync(context, tempFiles, cancellationToken);
+                Dictionary<string, PlaceFileResult> placeResults = await TryPlaceFilesFromCacheAsync(
+                    context,
+                    tempFiles,
+                    realizationModeOverride: FileRealizationMode.Any, // hard links are fine for these
+                    cancellationToken);
+
                 foreach (PlaceFileResult placeResult in placeResults.Values)
                 {
                     placeResult.ThrowIfFailure();
@@ -389,7 +394,11 @@ private static byte GetAlgorithmId(ContentHash hash)
         }
     }
 
-    private async Task<Dictionary<string, PlaceFileResult>> TryPlaceFilesFromCacheAsync(Context context, List<ContentHashWithPath> files, CancellationToken cancellationToken)
+    private async Task<Dictionary<string, PlaceFileResult>> TryPlaceFilesFromCacheAsync(
+        Context context,
+        List<ContentHashWithPath> files,
+        FileRealizationMode? realizationModeOverride,
+        CancellationToken cancellationToken)
     {
         // cache expects destination directories already exist
         foreach (ContentHashWithPath file in files)
@@ -400,7 +409,9 @@ private async Task<Dictionary<string, PlaceFileResult>> TryPlaceFilesFromCacheAs
         Dictionary<string, PlaceFileResult> results = new();
         List<ContentHashWithPath> places = new();
 
-        foreach (IGrouping<(byte algoId, FileRealizationMode mode), ContentHashWithPath>? filesGroup in files.GroupBy(f => (GetAlgorithmId(f.Hash), GetFileRealizationMode(f.Path.Path))))
+        var operationGroups = files.GroupBy(f => (GetAlgorithmId(f.Hash), realizationModeOverride ?? GetFileRealizationMode(f.Path.Path)));
+
+        foreach (IGrouping<(byte algoId, FileRealizationMode mode), ContentHashWithPath>? filesGroup in operationGroups)
         {
             FileRealizationMode realizationMode = filesGroup.Key.mode;
             FileAccessMode accessMode = realizationMode == FileRealizationMode.CopyNoVerify
@@ -500,7 +511,7 @@ public async Task PlaceFilesAsync(Context context, IReadOnlyDictionary<string, C
             // try to pull whole files from the cache
             var places = files.Select(f => new ContentHashWithPath(f.Value, new AbsolutePath(f.Key))).ToList();
 
-            Dictionary<string, PlaceFileResult> placeResults = await _client.TryPlaceFilesFromCacheAsync(context, places, cancellationToken);
+            Dictionary<string, PlaceFileResult> placeResults = await _client.TryPlaceFilesFromCacheAsync(context, places, realizationModeOverride: null, cancellationToken);
 
             Dictionary<string, ManifestItem> manifestItems = _manifest.Items.ToDictionary(i => Path.Combine(_client.RepoRoot, i.Path), i => i);
             var itemsToDownload = new List<ManifestItem>();