Merge pull request #159 from kyle-rader/feature/fast-fetch-force-flag

This adds a --force-checkout option when using FastFetch. This will cause FastFetch to treat the repo as if it had just been initialized meaning new folders can be fetched after the initial FastFetch is run. You can effectively grow the number of folders a FastFetched repo has.

This is required for boot-strapping scenarios where the set of desired folders to fetch must be dynamically computed after an initial fast fetch.

GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs → GVFS/FastFetch/CheckoutJob.cs

@@ -1,6 +1,8 @@
-using GVFS.Common.FileSystem;
+using GVFS.Common;
+using GVFS.Common.FileSystem;
 using GVFS.Common.Git;
 using GVFS.Common.Prefetch.Git;
+using GVFS.Common.Prefetch.Jobs;
 using GVFS.Common.Tracing;
 using System;
 using System.Collections.Concurrent;
@@ -9,7 +11,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 
-namespace GVFS.Common.Prefetch.Jobs
+namespace FastFetch
 {
     public class CheckoutJob : Job
     {
@@ -19,6 +21,7 @@ public class CheckoutJob : Job
         private ITracer tracer;
         private Enlistment enlistment;
         private string targetCommitSha;
+        private bool forceCheckout;
 
         private DiffHelper diff;
 
@@ -31,13 +34,14 @@ public class CheckoutJob : Job
         // Checkout requires synchronization between the delete/directory/add stages, so control the parallelization
         private int maxParallel;
 
-        public CheckoutJob(int maxParallel, IEnumerable<string> folderList, string targetCommitSha, ITracer tracer, Enlistment enlistment)
+        public CheckoutJob(int maxParallel, IEnumerable<string> folderList, string targetCommitSha, ITracer tracer, Enlistment enlistment, bool forceCheckout)
             : base(1)
         {
             this.tracer = tracer.StartActivity(AreaPath, EventLevel.Informational, Keywords.Telemetry, metadata: null);
             this.enlistment = enlistment;
             this.diff = new DiffHelper(tracer, enlistment, new string[0], folderList);
             this.targetCommitSha = targetCommitSha;
+            this.forceCheckout = forceCheckout;
             this.AvailableBlobShas = new BlockingCollection<string>();
 
             // Keep track of how parallel we're expected to be later during DoWork
@@ -62,7 +66,17 @@ public bool UpdatedWholeTree
 
         protected override void DoBeforeWork()
         {
-            this.diff.PerformDiff(this.targetCommitSha);
+            if (this.forceCheckout)
+            {
+                // Force search the entire tree by treating the repo as if it were brand new.
+                this.diff.PerformDiff(sourceTreeSha: null, targetTreeSha: this.targetCommitSha);
+            }
+            else
+            {
+                // Let the diff find the sourceTreeSha on its own.
+                this.diff.PerformDiff(this.targetCommitSha);
+            }
+
             this.HasFailures = this.diff.HasFailures;
         }
 

GVFS/FastFetch/CheckoutPrefetcher.cs

@@ -14,8 +14,9 @@ namespace FastFetch
 {
     public class CheckoutPrefetcher : BlobPrefetcher
     {
-        private readonly bool allowIndexMetadataUpdateFromWorkingTree;
         private readonly int checkoutThreadCount;
+        private readonly bool allowIndexMetadataUpdateFromWorkingTree;
+        private readonly bool forceCheckout;
 
         public CheckoutPrefetcher(
             ITracer tracer,
@@ -26,10 +27,12 @@ public CheckoutPrefetcher(
             int downloadThreadCount,
             int indexThreadCount,
             int checkoutThreadCount,
-            bool allowIndexMetadataUpdateFromWorkingTree) : base(tracer, enlistment, objectRequestor, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount)
+            bool allowIndexMetadataUpdateFromWorkingTree,
+            bool forceCheckout) : base(tracer, enlistment, objectRequestor, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount)
         {
             this.checkoutThreadCount = checkoutThreadCount;
             this.allowIndexMetadataUpdateFromWorkingTree = allowIndexMetadataUpdateFromWorkingTree;
+            this.forceCheckout = forceCheckout;
         }
 
         /// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
@@ -66,7 +69,7 @@ public override void Prefetch(string branchOrCommit, bool isBranch)
             // Configure pipeline
             // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper
             // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
-            CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment);
+            CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment, this.forceCheckout);
             FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
             BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
             IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);

GVFS/FastFetch/FastFetch.csproj

@@ -56,6 +56,7 @@
     <Compile Include="$(BuildOutputDir)\CommonAssemblyVersion.cs">
       <Link>CommonAssemblyVersion.cs</Link>
     </Compile>
+    <Compile Include="CheckoutJob.cs" />
     <Compile Include="CheckoutPrefetcher.cs" />
     <Compile Include="FastFetchVerb.cs" />
     <Compile Include="..\GVFS.PlatformLoader\PlatformLoader.Windows.cs" />

GVFS/FastFetch/FastFetchVerb.cs

@@ -54,6 +54,15 @@ public class FastFetchVerb
             HelpText = "Checkout the target commit into the working directory after fetching")]
         public bool Checkout { get; set; }
 
+        [Option(
+            "force-checkout",
+            Required = false,
+            Default = false,
+            HelpText = "Force FastFetch to checkout content as if the current repo had just been initialized." +
+                       "This allows you to include more folders from the repo that were not originally checked out." +
+                       "Can only be used with the --checkout option.")]
+        public bool ForceCheckout { get; set; }
+
         [Option(
             "search-thread-count",
             Required = false,
@@ -157,6 +166,12 @@ private int ExecuteWithExitCode()
                 Console.WriteLine("Cannot specify both a commit sha and a branch name.");
                 return ExitFailure;
             }
+
+            if (this.ForceCheckout && !this.Checkout)
+            {
+                Console.WriteLine("Cannot use --force-checkout option without --checkout option.");
+                return ExitFailure;
+            }
 
             this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount;
             this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads);
@@ -318,7 +333,8 @@ private BlobPrefetcher GetFolderPrefetcher(ITracer tracer, Enlistment enlistment
                     this.DownloadThreadCount,
                     this.IndexThreadCount,
                     this.CheckoutThreadCount,
-                    this.AllowIndexMetadataUpdateFromWorkingTree);
+                    this.AllowIndexMetadataUpdateFromWorkingTree,
+                    this.ForceCheckout);
             }
             else
             {

GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs

@@ -217,8 +217,7 @@ public void PrefetchWithStats(
 
             string previousCommit = null;
 
-            // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check
-            DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList);
+            // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check.
             if (File.Exists(shallowFile))
             {
                 previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault();
@@ -230,6 +229,8 @@ public void PrefetchWithStats(
                 }
             }
 
+            DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList);
+
             ThreadStart performDiff = () =>
             {
                 blobEnumerator.PerformDiff(previousCommit, commitToFetch);

GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs

@@ -100,6 +100,54 @@ public void CanFetchAndCheckoutASingleFolderIntoEmptyGitRepo()
             this.AllFetchedFilePathsShouldPassCheck(path => path.StartsWith("GVFS", StringComparison.OrdinalIgnoreCase));
         }
 
+        [TestCase]
+        public void CanFetchAndCheckoutMultipleTimesUsingForceCheckoutFlag()
+        {
+            this.RunFastFetch($"--checkout --folders \"/GVFS\" -b {Settings.Default.Commitish}");
+
+            this.CurrentBranchShouldEqual(Settings.Default.Commitish);
+
+            this.fastFetchRepoRoot.ShouldBeADirectory(FileSystemRunner.DefaultRunner);
+            List<string> dirs = Directory.EnumerateFileSystemEntries(this.fastFetchRepoRoot).ToList();
+            dirs.SequenceEqual(new[]
+            {
+                Path.Combine(this.fastFetchRepoRoot, ".git"),
+                Path.Combine(this.fastFetchRepoRoot, "GVFS"),
+                Path.Combine(this.fastFetchRepoRoot, "GVFS.sln")
+            });
+
+            Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "GVFS"), "*", SearchOption.AllDirectories)
+                .Count()
+                .ShouldEqual(345);
+            this.AllFetchedFilePathsShouldPassCheck(path => path.StartsWith("GVFS", StringComparison.OrdinalIgnoreCase));
+
+            // Run a second time in the same repo on the same branch with more folders. 
+            this.RunFastFetch($"--checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish} --force-checkout");
+            dirs = Directory.EnumerateFileSystemEntries(this.fastFetchRepoRoot).ToList();
+            dirs.SequenceEqual(new[]
+            {
+                Path.Combine(this.fastFetchRepoRoot, ".git"),
+                Path.Combine(this.fastFetchRepoRoot, "GVFS"),
+                Path.Combine(this.fastFetchRepoRoot, "Scripts"),
+                Path.Combine(this.fastFetchRepoRoot, "GVFS.sln")
+            });
+            Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "Scripts"), "*", SearchOption.AllDirectories)
+                .Count()
+                .ShouldEqual(5);
+        }
+
+        [TestCase]
+        public void ForceCheckoutRequiresCheckout()
+        {
+            this.RunFastFetch($"--checkout --folders \"/Scripts\" -b {Settings.Default.Commitish}");
+
+            // Run a second time in the same repo on the same branch with more folders. 
+            string result = this.RunFastFetch($"--force-checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish}");
+
+            string[] expectedResults = new string[] { "Cannot use --force-checkout option without --checkout option." };
+            result.ShouldContain(expectedResults);
+        }
+
         [TestCase]
         public void FastFetchFolderWithOnlyOneFile()
         {