Add --force flag to fast fetch

GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs → GVFS/FastFetch/CheckoutJob.cs

@@ -1,6 +1,8 @@
-using GVFS.Common.FileSystem;
+using GVFS.Common;
+using GVFS.Common.FileSystem;
 using GVFS.Common.Git;
 using GVFS.Common.Prefetch.Git;
+using GVFS.Common.Prefetch.Jobs;
 using GVFS.Common.Tracing;
 using System;
 using System.Collections.Concurrent;
@@ -9,7 +11,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 
-namespace GVFS.Common.Prefetch.Jobs
+namespace FastFetch
 {
     public class CheckoutJob : Job
     {
@@ -19,6 +21,7 @@ public class CheckoutJob : Job
         private ITracer tracer;
         private Enlistment enlistment;
         private string targetCommitSha;
+        private bool forceCheckout;
 
         private DiffHelper diff;
 
@@ -31,13 +34,14 @@ public class CheckoutJob : Job
         // Checkout requires synchronization between the delete/directory/add stages, so control the parallelization
         private int maxParallel;
 
-        public CheckoutJob(int maxParallel, IEnumerable<string> folderList, string targetCommitSha, ITracer tracer, Enlistment enlistment)
+        public CheckoutJob(int maxParallel, IEnumerable<string> folderList, string targetCommitSha, ITracer tracer, Enlistment enlistment, bool forceCheckout)
             : base(1)
         {
             this.tracer = tracer.StartActivity(AreaPath, EventLevel.Informational, Keywords.Telemetry, metadata: null);
             this.enlistment = enlistment;
             this.diff = new DiffHelper(tracer, enlistment, new string[0], folderList);
             this.targetCommitSha = targetCommitSha;
+            this.forceCheckout = forceCheckout;
             this.AvailableBlobShas = new BlockingCollection<string>();
 
             // Keep track of how parallel we're expected to be later during DoWork
@@ -62,7 +66,17 @@ public bool UpdatedWholeTree
 
         protected override void DoBeforeWork()
         {
-            this.diff.PerformDiff(this.targetCommitSha);
+            if (this.forceCheckout)
+            {
+                // Force search the entire tree by treating the repo as if it were brand new.
+                this.diff.PerformDiff(sourceTreeSha: null, targetTreeSha: this.targetCommitSha);
+            }
+            else
+            {
+                // Let the diff find the sourceTreeSha on its own.
+                this.diff.PerformDiff(this.targetCommitSha);
+            }
+
             this.HasFailures = this.diff.HasFailures;
         }
 

GVFS/FastFetch/CheckoutPrefetcher.cs

@@ -14,8 +14,9 @@ namespace FastFetch
 {
     public class CheckoutPrefetcher : BlobPrefetcher
     {
-        private readonly bool allowIndexMetadataUpdateFromWorkingTree;
         private readonly int checkoutThreadCount;
+        private readonly bool allowIndexMetadataUpdateFromWorkingTree;
+        private readonly bool forceCheckout;
 
         public CheckoutPrefetcher(
             ITracer tracer,
@@ -26,10 +27,12 @@ public CheckoutPrefetcher(
             int downloadThreadCount,
             int indexThreadCount,
             int checkoutThreadCount,
-            bool allowIndexMetadataUpdateFromWorkingTree) : base(tracer, enlistment, objectRequestor, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount)
+            bool allowIndexMetadataUpdateFromWorkingTree,
+            bool forceCheckout) : base(tracer, enlistment, objectRequestor, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount)
         {
             this.checkoutThreadCount = checkoutThreadCount;
             this.allowIndexMetadataUpdateFromWorkingTree = allowIndexMetadataUpdateFromWorkingTree;
+            this.forceCheckout = forceCheckout;
         }
 
         /// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
@@ -66,7 +69,7 @@ public override void Prefetch(string branchOrCommit, bool isBranch)
             // Configure pipeline
             // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper
             // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
-            CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment);
+            CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment, this.forceCheckout);
             FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
             BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
             IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);

GVFS/FastFetch/FastFetch.csproj

@@ -56,6 +56,7 @@
     <Compile Include="$(BuildOutputDir)\CommonAssemblyVersion.cs">
       <Link>CommonAssemblyVersion.cs</Link>
     </Compile>
+    <Compile Include="CheckoutJob.cs" />
     <Compile Include="CheckoutPrefetcher.cs" />
     <Compile Include="FastFetchVerb.cs" />
     <Compile Include="..\GVFS.PlatformLoader\PlatformLoader.Windows.cs" />

GVFS/FastFetch/FastFetchVerb.cs

@@ -54,6 +54,15 @@ public class FastFetchVerb
             HelpText = "Checkout the target commit into the working directory after fetching")]
         public bool Checkout { get; set; }
 
+        [Option(
+            "force-checkout",
+            Required = false,
+            Default = false,
+            HelpText = "Force FastFetch to checkout content as if the current repo had just been initialized." +
+                       "This allows you to include more folders from the repo that were not originally checked out." +
+                       "Can only be used with the --checkout option.")]
+        public bool ForceCheckout { get; set; }
+
         [Option(
             "search-thread-count",
             Required = false,
@@ -157,6 +166,12 @@ private int ExecuteWithExitCode()
                 Console.WriteLine("Cannot specify both a commit sha and a branch name.");
                 return ExitFailure;
             }
+
+            if (this.ForceCheckout && !this.Checkout)
+            {
+                Console.WriteLine("Cannot use --force-checkout option without --checkout option.");
+                return ExitFailure;
+            }
 
             this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount;
             this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads);
@@ -318,7 +333,8 @@ private BlobPrefetcher GetFolderPrefetcher(ITracer tracer, Enlistment enlistment
                     this.DownloadThreadCount,
                     this.IndexThreadCount,
                     this.CheckoutThreadCount,
-                    this.AllowIndexMetadataUpdateFromWorkingTree);
+                    this.AllowIndexMetadataUpdateFromWorkingTree,
+                    this.ForceCheckout);
             }
             else
             {

GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs

@@ -217,8 +217,7 @@ public void PrefetchWithStats(
 
             string previousCommit = null;
 
-            // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check
-            DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList);
+            // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check.
             if (File.Exists(shallowFile))
             {
                 previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault();
@@ -230,6 +229,8 @@ public void PrefetchWithStats(
                 }
             }
 
+            DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList);
+
             ThreadStart performDiff = () =>
             {
                 blobEnumerator.PerformDiff(previousCommit, commitToFetch);

GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs

@@ -100,6 +100,54 @@ public void CanFetchAndCheckoutASingleFolderIntoEmptyGitRepo()
             this.AllFetchedFilePathsShouldPassCheck(path => path.StartsWith("GVFS", StringComparison.OrdinalIgnoreCase));
         }
 
+        [TestCase]
+        public void CanFetchAndCheckoutMultipleTimesUsingForceCheckoutFlag()
+        {
+            this.RunFastFetch($"--checkout --folders \"/GVFS\" -b {Settings.Default.Commitish}");
+
+            this.CurrentBranchShouldEqual(Settings.Default.Commitish);
+
+            this.fastFetchRepoRoot.ShouldBeADirectory(FileSystemRunner.DefaultRunner);
+            List<string> dirs = Directory.EnumerateFileSystemEntries(this.fastFetchRepoRoot).ToList();
+            dirs.SequenceEqual(new[]
+            {
+                Path.Combine(this.fastFetchRepoRoot, ".git"),
+                Path.Combine(this.fastFetchRepoRoot, "GVFS"),
+                Path.Combine(this.fastFetchRepoRoot, "GVFS.sln")
+            });
+
+            Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "GVFS"), "*", SearchOption.AllDirectories)
+                .Count()
+                .ShouldEqual(345);
+            this.AllFetchedFilePathsShouldPassCheck(path => path.StartsWith("GVFS", StringComparison.OrdinalIgnoreCase));
+
+            // Run a second time in the same repo on the same branch with more folders. 
+            this.RunFastFetch($"--checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish} --force-checkout");
+            dirs = Directory.EnumerateFileSystemEntries(this.fastFetchRepoRoot).ToList();
+            dirs.SequenceEqual(new[]
+            {
+                Path.Combine(this.fastFetchRepoRoot, ".git"),
+                Path.Combine(this.fastFetchRepoRoot, "GVFS"),
+                Path.Combine(this.fastFetchRepoRoot, "Scripts"),
+                Path.Combine(this.fastFetchRepoRoot, "GVFS.sln")
+            });
+            Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "Scripts"), "*", SearchOption.AllDirectories)
+                .Count()
+                .ShouldEqual(5);
+        }
+
+        [TestCase]
+        public void ForceCheckoutRequiresCheckout()
+        {
+            this.RunFastFetch($"--checkout --folders \"/Scripts\" -b {Settings.Default.Commitish}");
+
+            // Run a second time in the same repo on the same branch with more folders. 
+            string result = this.RunFastFetch($"--force-checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish}");
+
+            string[] expectedResults = new string[] { "Cannot use --force-checkout option without --checkout option." };
+            result.ShouldContain(expectedResults);
+        }
+
         [TestCase]
         public void FastFetchFolderWithOnlyOneFile()
         {