Merge pull request #84: Run GitRepoTests in sparse mode

Resolves #77.

Update the functional tests to initialize a sparse enlistment with `scalar sparse --add-stdin` and to test lots of Git commands in that mode versus a vanilla Git repo. Uses the SparseMode checks from microsoft/vfsforgit to verify the contents _inside the cone_ match, but ignores all changes outside the cone. 

This took a lot longer than anticipated. Here are a few things that happened along the way:

* The untracked cache doesn't work correctly on Windows. The functional tests were failing due to quickly running status after every call. This _was_ working correctly on Mac, so maybe we re-enable it on a platform-specific basis later. Reverts #86 until we can revisit.

* The clone verb was causing problems. This is due to some timing concerns, mostly: VFS for Git needs a checkout to construct an index before mounting, but we can't checkout before mounting. Since the sparse verb doesn't prefetch before checkout (it doesn't need _everything_, but needs _some_ things), we needed to change this order. However, the `CloneVerb` code is very convoluted. It required a lot of refactoring to split out that checkout code. Hopefully the new code is cleaner and easier to modify.

* The input for the cone matches is slightly different for Git: we need `/` as the path separator, but the logic for interpreting those paths in the test code uses the platform-specific separator.

* The parallel test scripts were causing new issues in the `LooseObjectStepTests`. Turns out that in #68, I messed up the base constructor parameters and didn't give that enlistment a local object cache. That meant that other clones had prefetches and loose object downloads causing different numbers in these tests. That's fixed.

* `CloneWithDefaultLocalCacheLocation()` was failing on the build machine but not on my laptop. The error appeared to be due to failing to register the mount with the service. Turns out, this test used to use the `--no-mount` option, but that was removed in #80. Oops! Maybe we'll put that back as an `--internal_use_only` parameter. OR we can figure out why the service isn't running during that test.

Scalar.FunctionalTests/Program.cs

@@ -34,18 +34,17 @@ public static void Main(string[] args)
             HashSet<string> includeCategories = new HashSet<string>();
             HashSet<string> excludeCategories = new HashSet<string>();
 
+            // Run all GitRepoTests with sparse mode
+            ScalarTestConfig.GitRepoTestsValidateWorkTree =
+                new object[]
+                {
+                        new object[] { Settings.ValidateWorkingTreeMode.SparseMode },
+                };
+
             if (runner.HasCustomArg("--full-suite"))
             {
                 Console.WriteLine("Running the full suite of tests");
 
-                List<object[]> modes = new List<object[]>();
-                foreach (Settings.ValidateWorkingTreeMode mode in Enum.GetValues(typeof(Settings.ValidateWorkingTreeMode)))
-                {
-                    modes.Add(new object[] { mode });
-                }
-
-                ScalarTestConfig.GitRepoTestsValidateWorkTree = modes.ToArray();
-
                 if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 {
                     ScalarTestConfig.FileSystemRunners = FileSystemRunners.FileSystemRunner.AllWindowsRunners;
@@ -57,22 +56,6 @@ public static void Main(string[] args)
             }
             else
             {
-                Settings.ValidateWorkingTreeMode validateMode = Settings.ValidateWorkingTreeMode.Full;
-
-                if (runner.HasCustomArg("--sparse-mode"))
-                {
-                    validateMode = Settings.ValidateWorkingTreeMode.SparseMode;
-
-                    // Only test the git commands in sparse mode for splitting out tests in builds
-                    includeCategories.Add(Categories.GitCommands);
-                }
-
-                ScalarTestConfig.GitRepoTestsValidateWorkTree =
-                    new object[]
-                    {
-                        new object[] { validateMode },
-                    };
-
                 if (runner.HasCustomArg("--extra-only"))
                 {
                     Console.WriteLine("Running only the tests marked as ExtraCoverage");
@@ -95,13 +78,15 @@ public static void Main(string[] args)
                 includeCategories.Remove(Categories.ExtraCoverage);
             }
 
+            // Not just Mac, but no platform has status cache.
+            excludeCategories.Add(Categories.MacTODO.NeedsStatusCache);
+
             if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
             {
                 excludeCategories.Add(Categories.MacTODO.NeedsNewFolderCreateNotification);
                 excludeCategories.Add(Categories.MacTODO.NeedsScalarConfig);
                 excludeCategories.Add(Categories.MacTODO.NeedsDehydrate);
                 excludeCategories.Add(Categories.MacTODO.NeedsServiceVerb);
-                excludeCategories.Add(Categories.MacTODO.NeedsStatusCache);
                 excludeCategories.Add(Categories.MacTODO.TestNeedsToLockFile);
                 excludeCategories.Add(Categories.WindowsOnly);
             }

Scalar.FunctionalTests/Settings.cs

@@ -23,9 +23,6 @@ public static class Default
             public static string Commitish { get; set; }
             public static string ControlGitRepoRoot { get; set; }
             public static string EnlistmentRoot { get; set; }
-            public static string FastFetchBaseRoot { get; set; }
-            public static string FastFetchRoot { get; set; }
-            public static string FastFetchControl { get; set; }
             public static string PathToGit { get; set; }
             public static string PathToScalarService { get; set; }
             public static string BinaryFileNameExtension { get; set; }
@@ -45,9 +42,6 @@ public static void Initialize()
                     PathToBash = @"C:\Program Files\Git\bin\bash.exe";
 
                     ControlGitRepoRoot = @"C:\Repos\ScalarFunctionalTests\ControlRepo";
-                    FastFetchBaseRoot = @"C:\Repos\ScalarFunctionalTests\FastFetch";
-                    FastFetchRoot = Path.Combine(FastFetchBaseRoot, "test");
-                    FastFetchControl = Path.Combine(FastFetchBaseRoot, "control");
                     PathToScalarService = @"Scalar.Service.exe";
                     BinaryFileNameExtension = ".exe";
                 }
@@ -58,9 +52,6 @@ public static void Initialize()
                         "Scalar.FT");
                     EnlistmentRoot = Path.Combine(root, "test");
                     ControlGitRepoRoot = Path.Combine(root, "control");
-                    FastFetchBaseRoot = Path.Combine(root, "FastFetch");
-                    FastFetchRoot = Path.Combine(FastFetchBaseRoot, "test");
-                    FastFetchControl = Path.Combine(FastFetchBaseRoot, "control");
                     PathToScalar = "scalar";
                     PathToGit = "/usr/local/bin/git";
                     PathToBash = "/bin/bash";

Scalar.FunctionalTests/Should/FileSystemShouldExtensions.cs

@@ -60,9 +60,6 @@ public static string ShouldNotExistOnDisk(this string path, FileSystemRunner run
 
         public class FileAdapter
         {
-            private const int MaxWaitMS = 2000;
-            private const int ThreadSleepMS = 100;
-
             private FileSystemRunner runner;
 
             public FileAdapter(string path, FileSystemRunner runner)
@@ -260,6 +257,18 @@ private static bool IsMatchedPath(FileSystemInfo info, string repoRoot, string[]
                 }
 
                 string localPath = info.FullName.Substring(repoRoot.Length + 1);
+                int parentLength = localPath.LastIndexOf(System.IO.Path.DirectorySeparatorChar);
+
+                string parentPath = null;
+
+                if ((info.Attributes & FileAttributes.Directory) == FileAttributes.Directory)
+                {
+                    parentPath = localPath;
+                }
+                else if (parentLength > 0)
+                {
+                    parentPath = localPath.Substring(0, parentLength + 1);
+                }
 
                 if (localPath.Equals(".git", StringComparison.OrdinalIgnoreCase))
                 {
@@ -268,7 +277,7 @@ private static bool IsMatchedPath(FileSystemInfo info, string repoRoot, string[]
                     return true;
                 }
 
-                if (!localPath.Contains(System.IO.Path.DirectorySeparatorChar) &&
+                if (parentPath == null &&
                     (info.Attributes & FileAttributes.Directory) != FileAttributes.Directory)
                 {
                     // If it is a file in the root folder, then include it.
@@ -277,15 +286,15 @@ private static bool IsMatchedPath(FileSystemInfo info, string repoRoot, string[]
 
                 foreach (string prefixDir in prefixes)
                 {
-                    if (localPath.StartsWith(prefixDir, StringComparison.OrdinalIgnoreCase))
+                    if (localPath.Equals(prefixDir, StringComparison.OrdinalIgnoreCase) ||
+                        localPath.StartsWith(prefixDir + System.IO.Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase))
                     {
                         return true;
                     }
 
-                    if (prefixDir.StartsWith(localPath, StringComparison.OrdinalIgnoreCase) &&
-                        Directory.Exists(info.FullName))
+                    if (parentPath != null && prefixDir.StartsWith(parentPath, StringComparison.OrdinalIgnoreCase))
                     {
-                        // For example: localPath = "Scalar" and prefix is "Scalar\\Scalar".
+                        // For example: localPath = "Scalar\\file.txt", parentPath="Scalar\\" and prefix is "Scalar\\Scalar".
                         return true;
                     }
                 }
@@ -305,6 +314,7 @@ private static void CompareDirectories(
                 IEnumerable<FileSystemInfo> actualEntries = new DirectoryInfo(actualPath).EnumerateFileSystemInfos("*", SearchOption.AllDirectories);
 
                 string dotGitFolder = System.IO.Path.DirectorySeparatorChar + TestConstants.DotGit.Root + System.IO.Path.DirectorySeparatorChar;
+
                 IEnumerator<FileSystemInfo> expectedEnumerator = expectedEntries
                     .Where(x => !x.FullName.Contains(dotGitFolder))
                     .OrderBy(x => x.FullName)

Scalar.FunctionalTests/Tests/EnlistmentPerFixture/CloneTests.cs

@@ -48,6 +48,7 @@ public void CloneWithLocalCachePathWithinSrc()
 
         [TestCase]
         [Category(Categories.MacOnly)]
+        [Category(Categories.NeedsUpdatesForNonVirtualizedMode)]
         public void CloneWithDefaultLocalCacheLocation()
         {
             FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner;
@@ -57,11 +58,15 @@ public void CloneWithDefaultLocalCacheLocation()
             string newEnlistmentRoot = ScalarFunctionalTestEnlistment.GetUniqueEnlistmentRoot();
 
             ProcessStartInfo processInfo = new ProcessStartInfo(ScalarTestConfig.PathToScalar);
+
+            // Needs update for non-virtualized mode: this used to have --no-mount to avoid an issue
+            // with registering the mount with the service.
             processInfo.Arguments = $"clone {Properties.Settings.Default.RepoToClone} {newEnlistmentRoot} --no-prefetch";
             processInfo.WindowStyle = ProcessWindowStyle.Hidden;
             processInfo.CreateNoWindow = true;
             processInfo.UseShellExecute = false;
             processInfo.RedirectStandardOutput = true;
+            processInfo.WorkingDirectory = Properties.Settings.Default.EnlistmentRoot;
 
             ProcessResult result = ProcessHelper.Run(processInfo);
             result.ExitCode.ShouldEqual(0, result.Errors);

Scalar.FunctionalTests/Tests/EnlistmentPerFixture/LooseObjectStepTests.cs

@@ -18,7 +18,7 @@ public class LooseObjectStepTests : TestsWithEnlistmentPerFixture
         // Set forcePerRepoObjectCache to true to avoid any of the tests inadvertently corrupting
         // the cache
         public LooseObjectStepTests()
-            : base(forcePerRepoObjectCache: false)
+            : base(forcePerRepoObjectCache: true, skipPrefetchDuringClone: false)
         {
             this.fileSystem = new SystemIORunner();
         }
@@ -31,6 +31,7 @@ public LooseObjectStepTests()
         [Order(1)]
         public void NoLooseObjectsDoesNothing()
         {
+            this.Enlistment.UnmountScalar();
             this.DeleteFiles(this.GetLooseObjectFiles());
 
             this.DeleteFiles(this.GetLooseObjectFiles());