From 299b3620027bf0837a7a6d0774bc2b2a7f1c2654 Mon Sep 17 00:00:00 2001 From: Lessley Dennington Date: Fri, 17 Sep 2021 12:08:48 -0700 Subject: [PATCH 01/29] Upgrade `update-winget` action to v1.4 This version allows us to update existing `winget` manifests by specifying a manifest's SHA if it already exists in `microsoft/winget-pkgs`. Pull Requests --------------- * [#178](https://github.com/mjcheetham/update-winget/pull/179): Allow updates to existing winget manifests --- .github/workflows/release-winget.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-winget.yaml b/.github/workflows/release-winget.yaml index 710bc94e7..694440de9 100644 --- a/.github/workflows/release-winget.yaml +++ b/.github/workflows/release-winget.yaml @@ -9,7 +9,7 @@ jobs: steps: - id: update-winget name: Update winget repository - uses: mjcheetham/update-winget@v1.2.2 + uses: mjcheetham/update-winget@v1.4 with: id: Microsoft.VFSforGit token: ${{ secrets.WINGET_TOKEN }} From ffc90e117c16846e6526ade5349619dbb174ac70 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 5 Oct 2021 17:58:28 -0400 Subject: [PATCH 02/29] Readme: add new winget install instructions Signed-off-by: Derrick Stolee --- Readme.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index 3dc9dbfdd..749fbbc92 100644 --- a/Readme.md +++ b/Readme.md @@ -22,8 +22,19 @@ in Git, Scalar offers a clearer path forward for all large monorepos. ## Installing VFS for Git -* VFS for Git requires Windows 10 Anniversary Update (Windows 10 version 1607) or later -* Run the latest GVFS and Git for Windows installers from https://github.com/Microsoft/VFSForGit/releases +VFS for Git requires Windows 10 Anniversary Update (Windows 10 version 1607) or later. + +To install, use [`winget`](https://github.com/microsoft/winget) to install the +[`microsoft/git` fork of Git](https://github.com/microsoft/git) and VFS for Git +using: + +``` +winget install microsoft-git +winget install gvfs +``` + +You will need to continue using the `microsoft/git` version of Git, and it has +will notify you when new versions are available. ## Building VFS for Git From c846c9bfc3b1b8c59fb775d546dfc6a3949f77db Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 5 Oct 2021 18:07:20 -0400 Subject: [PATCH 03/29] Update Readme.md Co-authored-by: Victoria Dye --- Readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 749fbbc92..a05e3ad54 100644 --- a/Readme.md +++ b/Readme.md @@ -33,9 +33,10 @@ winget install microsoft-git winget install gvfs ``` -You will need to continue using the `microsoft/git` version of Git, and it has +You will need to continue using the `microsoft/git` version of Git, and it will notify you when new versions are available. + ## Building VFS for Git If you'd like to build your own VFS for Git Windows installer: From 2e1dafe2120268b85ee72aff7f6abe5b0feacbe5 Mon Sep 17 00:00:00 2001 From: Lessley Dennington Date: Tue, 12 Oct 2021 12:59:59 -0700 Subject: [PATCH 04/29] Support spaces in paths when compiling Updating necessary files to support spaces in paths when compiling VFS for Git. --- GVFS/GVFS.Installers/GVFS.Installers.csproj | 2 +- GVFS/GVFS.Payload/GVFS.Payload.csproj | 2 +- GVFS/GVFS.Payload/layout.bat | 2 +- scripts/Build.bat | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.Installers/GVFS.Installers.csproj b/GVFS/GVFS.Installers/GVFS.Installers.csproj index da65ffbe4..1f5b692d1 100644 --- a/GVFS/GVFS.Installers/GVFS.Installers.csproj +++ b/GVFS/GVFS.Installers/GVFS.Installers.csproj @@ -41,7 +41,7 @@ - + diff --git a/GVFS/GVFS.Payload/GVFS.Payload.csproj b/GVFS/GVFS.Payload/GVFS.Payload.csproj index a976360f6..1df5a3210 100644 --- a/GVFS/GVFS.Payload/GVFS.Payload.csproj +++ b/GVFS/GVFS.Payload/GVFS.Payload.csproj @@ -29,7 +29,7 @@ - + diff --git a/GVFS/GVFS.Payload/layout.bat b/GVFS/GVFS.Payload/layout.bat index 85036a501..de1c0df81 100644 --- a/GVFS/GVFS.Payload/layout.bat +++ b/GVFS/GVFS.Payload/layout.bat @@ -38,7 +38,7 @@ SET VCRUNTIME=%4 SET OUTPUT=%5 SET ROOT=%~dp0..\.. -SET BUILD_OUT=%ROOT%\..\out +SET BUILD_OUT="%ROOT%\..\out" SET MANAGED_OUT_FRAGMENT=bin\%CONFIGURATION%\net461\win-x64 SET NATIVE_OUT_FRAGMENT=bin\x64\%CONFIGURATION% diff --git a/scripts/Build.bat b/scripts/Build.bat index a8f263e78..637b1e18e 100644 --- a/scripts/Build.bat +++ b/scripts/Build.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 +CALL "%~dp0\InitializeEnvironment.bat" || EXIT /b 10 SETLOCAL SETLOCAL EnableDelayedExpansion @@ -44,7 +44,7 @@ IF NOT EXIST "%NUGET_EXEC%" ( REM Acquire vswhere to find VS installations reliably SET VSWHERE_VER=2.6.7 "%NUGET_EXEC%" install vswhere -Version %VSWHERE_VER% || exit /b 1 -SET VSWHERE_EXEC=%VFS_PACKAGESDIR%\vswhere.%VSWHERE_VER%\tools\vswhere.exe +SET VSWHERE_EXEC="%VFS_PACKAGESDIR%\vswhere.%VSWHERE_VER%\tools\vswhere.exe" REM Assumes default installation location for Windows 10 SDKs IF NOT EXIST "C:\Program Files (x86)\Windows Kits\10\Include\10.0.16299.0" ( @@ -69,7 +69,7 @@ IF NOT DEFINED MSBUILD_EXEC ( ECHO ^********************** ECHO ^* Restoring Packages * ECHO ^********************** -"%MSBUILD_EXEC%" %VFS_SRCDIR%\GVFS.sln ^ +"%MSBUILD_EXEC%" "%VFS_SRCDIR%\GVFS.sln" ^ /t:Restore ^ /v:%VERBOSITY% ^ /p:Configuration=%CONFIGURATION% || GOTO ERROR @@ -77,7 +77,7 @@ ECHO ^********************** ECHO ^********************* ECHO ^* Building Solution * ECHO ^********************* -"%MSBUILD_EXEC%" %VFS_SRCDIR%\GVFS.sln ^ +"%MSBUILD_EXEC%" "%VFS_SRCDIR%\GVFS.sln" ^ /t:Build ^ /v:%VERBOSITY% ^ /p:Configuration=%CONFIGURATION% || GOTO ERROR From 35f2885c4710a0e9fa0340ef8d89cc192ceb4958 Mon Sep 17 00:00:00 2001 From: Lessley Dennington Date: Tue, 12 Oct 2021 14:36:27 -0700 Subject: [PATCH 05/29] Update inno installer metadata Updating inno installer metadata to align with the specification outlined in github/git-fundamentals#700. --- GVFS/GVFS.Installers/Setup.iss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Installers/Setup.iss b/GVFS/GVFS.Installers/Setup.iss index f12f289fc..ca93ac914 100644 --- a/GVFS/GVFS.Installers/Setup.iss +++ b/GVFS/GVFS.Installers/Setup.iss @@ -3,9 +3,9 @@ ; General documentation on how to use InnoSetup scripts: http://www.jrsoftware.org/ishelp/index.php -#define MyAppName "GVFS" +#define MyAppName "VFS for Git" #define MyAppInstallerVersion GetFileVersion(LayoutDir + "\GVFS.exe") -#define MyAppPublisher "Microsoft Corporation" +#define MyAppPublisher "Microsoft" #define MyAppPublisherURL "http://www.microsoft.com" #define MyAppURL "https://github.com/microsoft/VFSForGit" #define MyAppExeName "GVFS.exe" From 68d7d38f1012604d4a1635b7b49e2f376a3bdb21 Mon Sep 17 00:00:00 2001 From: Lessley Dennington Date: Tue, 12 Oct 2021 15:07:11 -0700 Subject: [PATCH 06/29] Run release-winget on Windows Server `wingetcreate` is a Windows exe and thus must be run on Windows. Updating our release-winget workflow accordingly. --- .github/workflows/release-winget.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-winget.yaml b/.github/workflows/release-winget.yaml index 694440de9..8763e4acc 100644 --- a/.github/workflows/release-winget.yaml +++ b/.github/workflows/release-winget.yaml @@ -5,7 +5,7 @@ on: jobs: release: - runs-on: ubuntu-latest + runs-on: windows-latest steps: - id: update-winget name: Update winget repository From 80a3812311663419fef1571b23229a46d38009b2 Mon Sep 17 00:00:00 2001 From: Eschryn <8217655+Eschryn@users.noreply.github.com> Date: Fri, 15 Oct 2021 11:16:23 +0200 Subject: [PATCH 07/29] "winget install microsoft-git" finds entries in msstore Adding the source parameter will make sure that the install command will actually install from winget --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index a05e3ad54..2f9401a51 100644 --- a/Readme.md +++ b/Readme.md @@ -29,7 +29,7 @@ To install, use [`winget`](https://github.com/microsoft/winget) to install the using: ``` -winget install microsoft-git +winget install -s winget microsoft-git winget install gvfs ``` From 96ea5d461935cec677c2a5aad669851a1eede82b Mon Sep 17 00:00:00 2001 From: Eschryn <8217655+Eschryn@users.noreply.github.com> Date: Fri, 15 Oct 2021 19:01:19 +0200 Subject: [PATCH 08/29] Update Readme.md --- Readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index 2f9401a51..10a28ca8d 100644 --- a/Readme.md +++ b/Readme.md @@ -29,8 +29,8 @@ To install, use [`winget`](https://github.com/microsoft/winget) to install the using: ``` -winget install -s winget microsoft-git -winget install gvfs +winget install --id Microsoft.Git +winget install --id Microsoft.VFSforGit ``` You will need to continue using the `microsoft/git` version of Git, and it From 91c0292498946321547295a0ba1f4f9a2b194d0d Mon Sep 17 00:00:00 2001 From: Anton Pegushin Date: Wed, 27 Oct 2021 13:19:35 -0700 Subject: [PATCH 09/29] Update winget link to a reference winget-cli repo. --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 10a28ca8d..b0578ff4f 100644 --- a/Readme.md +++ b/Readme.md @@ -24,7 +24,7 @@ in Git, Scalar offers a clearer path forward for all large monorepos. VFS for Git requires Windows 10 Anniversary Update (Windows 10 version 1607) or later. -To install, use [`winget`](https://github.com/microsoft/winget) to install the +To install, use [`winget`](https://github.com/microsoft/winget-cli) to install the [`microsoft/git` fork of Git](https://github.com/microsoft/git) and VFS for Git using: From da04504180b1140209525730fd4e1fcece5b3cfa Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sat, 30 Oct 2021 14:38:41 -0400 Subject: [PATCH 10/29] Update Git to v2.34.0.vfs.0.0 --- Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Version.props b/Version.props index 09487168e..6ee26e84b 100644 --- a/Version.props +++ b/Version.props @@ -18,7 +18,7 @@ including taking version numbers 2.X.Y from upstream and updating .W if we have any hotfixes to microsoft/git. --> - 2.20210817.4 + 2.20211115.1 v2.31.0.vfs.0.1 From 0a80e96a3e8c6caee3b55476fc7ab3c9dd4cdfb5 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 1 Nov 2021 12:47:20 -0400 Subject: [PATCH 11/29] Move expected path from "GVFS" to "VFS for Git" --- GVFS/GVFS.FunctionalTests/Settings.cs | 4 ++-- GVFS/GVFS.FunctionalTests/Tools/ProjFSFilterInstaller.cs | 2 +- GVFS/GVFS.FunctionalTests/Windows/Tests/ServiceTests.cs | 2 +- GVFS/GVFS.Installers/info.bat | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Settings.cs b/GVFS/GVFS.FunctionalTests/Settings.cs index a2b287163..1abc3b851 100644 --- a/GVFS/GVFS.FunctionalTests/Settings.cs +++ b/GVFS/GVFS.FunctionalTests/Settings.cs @@ -45,7 +45,7 @@ public static void Initialize() Commitish = @"FunctionalTests/20201014"; EnlistmentRoot = @"C:\Repos\GVFSFunctionalTests\enlistment"; - PathToGVFS = @"C:\Program Files\GVFS\GVFS.exe"; + PathToGVFS = @"C:\Program Files\VFS for Git\GVFS.exe"; PathToGit = @"C:\Program Files\Git\cmd\git.exe"; PathToBash = @"C:\Program Files\Git\bin\bash.exe"; @@ -53,7 +53,7 @@ public static void Initialize() FastFetchBaseRoot = @"C:\Repos\GVFSFunctionalTests\FastFetch"; FastFetchRoot = Path.Combine(FastFetchBaseRoot, "test"); FastFetchControl = Path.Combine(FastFetchBaseRoot, "control"); - PathToGVFSService = @"C:\Program Files\GVFS\GVFS.Service.exe"; + PathToGVFSService = @"C:\Program Files\VFS for Git\GVFS.Service.exe"; BinaryFileNameExtension = ".exe"; } } diff --git a/GVFS/GVFS.FunctionalTests/Tools/ProjFSFilterInstaller.cs b/GVFS/GVFS.FunctionalTests/Tools/ProjFSFilterInstaller.cs index c56ea8f38..71b471728 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/ProjFSFilterInstaller.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/ProjFSFilterInstaller.cs @@ -9,7 +9,7 @@ public class ProjFSFilterInstaller private const string GVFSServiceName = "GVFS.Service"; private const string ProjFSServiceName = "prjflt"; private const string OptionalFeatureName = "Client-ProjFS"; - private const string GVFSInstallPath = @"C:\Program Files\GVFS"; + private const string GVFSInstallPath = @"C:\Program Files\VFS for Git"; private const string NativeProjFSLibInstallLocation = GVFSInstallPath + @"\ProjFS\ProjectedFSLib.dll"; private const string PrjfltInfName = "prjflt.inf"; diff --git a/GVFS/GVFS.FunctionalTests/Windows/Tests/ServiceTests.cs b/GVFS/GVFS.FunctionalTests/Windows/Tests/ServiceTests.cs index ddbca48c1..18b705c0b 100644 --- a/GVFS/GVFS.FunctionalTests/Windows/Tests/ServiceTests.cs +++ b/GVFS/GVFS.FunctionalTests/Windows/Tests/ServiceTests.cs @@ -16,7 +16,7 @@ namespace GVFS.FunctionalTests.Windows.Tests [Category(Categories.ExtraCoverage)] public class ServiceTests : TestsWithEnlistmentPerFixture { - private const string NativeLibPath = @"C:\Program Files\GVFS\ProjectedFSLib.dll"; + private const string NativeLibPath = @"C:\Program Files\VFS for Git\ProjectedFSLib.dll"; private const string PrjFltAutoLoggerKey = "SYSTEM\\CurrentControlSet\\Control\\WMI\\Autologger\\Microsoft-Windows-ProjFS-Filter-Log"; private const string PrjFltAutoLoggerStartValue = "Start"; diff --git a/GVFS/GVFS.Installers/info.bat b/GVFS/GVFS.Installers/info.bat index 335fc269e..b068f9c69 100644 --- a/GVFS/GVFS.Installers/info.bat +++ b/GVFS/GVFS.Installers/info.bat @@ -3,10 +3,10 @@ SETLOCAL SET SYS_PRJFLT=C:\Windows\System32\drivers\prjflt.sys SET SYS_PROJFSLIB=C:\Windows\System32\ProjectedFSLib.dll -SET VFS_PROJFSLIB=C:\Program Files\GVFS\ProjectedFSLib.dll -SET VFS_BUND_PRJFLT=C:\Program Files\GVFS\Filter\PrjFlt.sys -SET VFS_BUND_PROJFSLIB=C:\Program Files\GVFS\ProjFS\ProjectedFSLib.dll -SET VFS_EXEC=C:\Program Files\GVFS\GVFS.exe +SET VFS_PROJFSLIB=C:\Program Files\VFS for Git\ProjectedFSLib.dll +SET VFS_BUND_PRJFLT=C:\Program Files\VFS for Git\Filter\PrjFlt.sys +SET VFS_BUND_PROJFSLIB=C:\Program Files\VFS for Git\ProjFS\ProjectedFSLib.dll +SET VFS_EXEC=C:\Program Files\VFS for Git\GVFS.exe SET GIT_EXEC=C:\Program Files\Git\cmd\git.exe ECHO Checking ProjFS Windows feature... From 416e3d324dcbfdc5932cc0b2aa7fb94f808c8f97 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 1 Nov 2021 14:25:16 -0400 Subject: [PATCH 12/29] install.bat: specify install dir --- .github/workflows/build.yaml | 2 +- GVFS/GVFS.Installers/install.bat | 2 +- scripts/RunFunctionalTests.bat | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8dca59006..a30be0879 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -109,7 +109,7 @@ jobs: - name: Run functional tests shell: cmd run: | - SET PATH=C:\Program Files\GVFS;%PATH% + SET PATH=C:\Program Files\VFS for Git;%PATH% SET GIT_TRACE2_PERF=C:\temp\git-trace2.log ft\GVFS.FunctionalTests.exe /result:TestResult.xml --ci diff --git a/GVFS/GVFS.Installers/install.bat b/GVFS/GVFS.Installers/install.bat index fb9299153..c629c75bc 100644 --- a/GVFS/GVFS.Installers/install.bat +++ b/GVFS/GVFS.Installers/install.bat @@ -16,4 +16,4 @@ ECHO Installing Git for Windows... %GIT_INSTALLER% /LOG="%LOGDIR%\git.log" /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /ALLOWDOWNGRADE=1 ECHO Installing VFS for Git... -%GVFS_INSTALLER% /LOG="%LOGDIR%\gvfs.log" /VERYSILENT /SUPPRESSMSGBOXES /NORESTART +%GVFS_INSTALLER% /LOG="%LOGDIR%\gvfs.log" /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /DIR="C:\Program Files\VFS for Git" diff --git a/scripts/RunFunctionalTests.bat b/scripts/RunFunctionalTests.bat index 7ad871217..80ab7e1e4 100644 --- a/scripts/RunFunctionalTests.bat +++ b/scripts/RunFunctionalTests.bat @@ -5,7 +5,7 @@ IF "%1"=="" (SET "CONFIGURATION=Debug") ELSE (SET "CONFIGURATION=%1") REM Ensure GVFS installation is on the PATH for the Functional Tests to find SETLOCAL -SET PATH=C:\Program Files\GVFS;C:\Program Files\Git\cmd;%PATH% +SET PATH=C:\Program Files\VFS for Git\;C:\Program Files\GVFS;C:\Program Files\Git\cmd;%PATH% ECHO PATH = %PATH% From 8f529a18933d85e6e595e866878211acd41e4301 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 1 Nov 2021 17:06:49 -0400 Subject: [PATCH 13/29] Remove flaky packfile maintenance tests These are verified by the Scalar functional tests, anyway. --- .../GVFS.FunctionalTests.csproj | 2 +- .../PackfileMaintenanceStepTests.cs | 134 ------------------ 2 files changed, 1 insertion(+), 135 deletions(-) delete mode 100644 GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PackfileMaintenanceStepTests.cs diff --git a/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj b/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj index eb0a937cc..345a70bcb 100644 --- a/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj +++ b/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj @@ -1,4 +1,4 @@ - + net461 diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PackfileMaintenanceStepTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PackfileMaintenanceStepTests.cs deleted file mode 100644 index 2a12eebc8..000000000 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PackfileMaintenanceStepTests.cs +++ /dev/null @@ -1,134 +0,0 @@ -using GVFS.FunctionalTests.FileSystemRunners; -using GVFS.FunctionalTests.Tools; -using GVFS.Tests.Should; -using NUnit.Framework; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture -{ - [TestFixture] - public class PackfileMaintenanceStepTests : TestsWithEnlistmentPerFixture - { - private FileSystemRunner fileSystem; - - // Set forcePerRepoObjectCache to true to avoid any of the tests inadvertently corrupting - // the cache - public PackfileMaintenanceStepTests() - : base(forcePerRepoObjectCache: true) - { - this.fileSystem = new SystemIORunner(); - } - - private string GitObjectRoot => this.Enlistment.GetObjectRoot(this.fileSystem); - private string PackRoot => this.Enlistment.GetPackRoot(this.fileSystem); - - [TestCase, Order(1)] - public void ExpireClonePack() - { - this.GetPackSizes(out int packCount, out long maxSize, out long minSize, out long totalSize); - - // We should have at least two packs: - // - // 1. the pack-.pack from clone. - // 2. a prefetch--.pack from prefetch. - // - // The prefetch pack is newer, and covers all the objects in the clone pack, - // so the clone pack will be expired when we run the step. - - Directory.GetFiles(this.PackRoot, "*.keep") - .Count() - .ShouldEqual(1); - - packCount.ShouldEqual(2, message: "Incorrect packfile layout for expire test"); - - // Ensure we have a multi-pack-index (not created on clone) - GitProcess.InvokeProcess( - this.Enlistment.RepoRoot, - $"multi-pack-index write --object-dir={this.GitObjectRoot}"); - - this.Enlistment.PackfileMaintenanceStep(); - - List packs = this.GetPackfiles(); - - packs.Count.ShouldEqual(1, $"incorrect number of packs after first step: {packs.Count}"); - - Path.GetFileName(packs[0]) - .StartsWith("prefetch-") - .ShouldBeTrue($"packsBetween[0] should start with 'prefetch-': {packs[0]}"); - } - - [TestCase, Order(2)] - public void RepackAllToOnePack() - { - // Create new pack(s) by prefetching blobs for a folder. - // This generates a number of packs, based on the processor number (for parallel downloads). - this.Enlistment.Prefetch($"--folders {Path.Combine("GVFS", "GVFS")}"); - - // Create a multi-pack-index that covers the prefetch packs - // (The post-fetch job creates a multi-pack-index only after a --commits prefetch) - GitProcess.InvokeProcess( - this.Enlistment.RepoRoot, - $"multi-pack-index write --object-dir={this.GitObjectRoot}"); - - // Run the step to ensure we don't have any packs that will be expired during the repack step - this.Enlistment.PackfileMaintenanceStep(); - - this.GetPackSizes(out int afterPrefetchPackCount, out long maxSize, out long minSize, out long totalSize); - - // Cannot be sure of the count, as the prefetch uses parallel threads to get multiple packs - afterPrefetchPackCount.ShouldBeAtLeast(2); - - this.Enlistment.PackfileMaintenanceStep(batchSize: totalSize - minSize + 1); - } - - [TestCase, Order(3)] - public void ExpireAllButOneAndKeep() - { - string prefetchPack = Directory.GetFiles(this.PackRoot, "prefetch-*.pack") - .FirstOrDefault(); - - prefetchPack.ShouldNotBeNull(); - - // We should expire all packs except the one we just created, - // and the prefetch pack which is marked as ".keep" - this.Enlistment.PackfileMaintenanceStep(); - - List packsAfter = this.GetPackfiles(); - - packsAfter.Count.ShouldEqual(2, $"incorrect number of packs after final expire step: {packsAfter.Count}"); - packsAfter.Contains(prefetchPack).ShouldBeTrue($"packsAfter does not contain prefetch pack ({prefetchPack})"); - } - - private List GetPackfiles() - { - return Directory.GetFiles(this.PackRoot, "*.pack").ToList(); - } - - private void GetPackSizes(out int packCount, out long maxSize, out long minSize, out long totalSize) - { - totalSize = 0; - maxSize = 0; - minSize = long.MaxValue; - packCount = 0; - - foreach (string file in this.GetPackfiles()) - { - packCount++; - long size = new FileInfo(Path.Combine(this.PackRoot, file)).Length; - totalSize += size; - - if (size > maxSize) - { - maxSize = size; - } - - if (size < minSize) - { - minSize = size; - } - } - } - } -} From c97079d727a7470104252270f5b3b8e2506b3f32 Mon Sep 17 00:00:00 2001 From: Lessley Dennington Date: Mon, 8 Nov 2021 11:08:38 -0800 Subject: [PATCH 14/29] Deploy to winget via wingetcreate Remove the custom mjcheetham/update-winget task in favor of the wingetcreate tool. --- .github/workflows/release-winget.yaml | 43 ++++++++------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/.github/workflows/release-winget.yaml b/.github/workflows/release-winget.yaml index 8763e4acc..2f4c79437 100644 --- a/.github/workflows/release-winget.yaml +++ b/.github/workflows/release-winget.yaml @@ -7,33 +7,16 @@ jobs: release: runs-on: windows-latest steps: - - id: update-winget - name: Update winget repository - uses: mjcheetham/update-winget@v1.4 - with: - id: Microsoft.VFSforGit - token: ${{ secrets.WINGET_TOKEN }} - releaseAsset: SetupGVFS.([0-9.]*)\.exe - manifestText: | - PackageIdentifier: {{id}} - PackageVersion: {{version}} - PackageName: VFS for Git - Publisher: Microsoft Corporation - Moniker: vfs-for-git - PackageUrl: https://aka.ms/vfs-for-git - Tags: - - vfs for git - - vfs-for-git - - vfsforgit - - gvfs - License: Copyright (C) Microsoft Corporation - ShortDescription: Virtual File System for Git - a tool to scale Git for monorepo scenarios. - Installers: - - Architecture: x64 - InstallerUrl: {{url}} - InstallerType: inno - InstallerSha256: {{sha256}} - PackageLocale: en-US - ManifestType: singleton - ManifestVersion: 1.0.0 - alwaysUsePullRequest: true \ No newline at end of file + - name: Publish manifest with winget-create + run: | + # Get correct release asset + $github = Get-Content '${{ github.event_path }}' | ConvertFrom-Json + $asset = $github.release.assets | Where-Object -Property name -match 'SetupGVFS[\d\.]*.exe' + + # Remove 'v' from the version + $version = $github.release.tag_name -replace ".v","" + + # Download and run wingetcreate + Invoke-WebRequest https://aka.ms/wingetcreate/latest -OutFile wingetcreate.exe + .\wingetcreate.exe update Microsoft.VFSforGit -u $asset.browser_download_url -v $version -o manifests -t "${{ secrets.WINGET_TOKEN }}" -s + shell: powershell \ No newline at end of file From e190f16237bfc2ca37d9033d8e1af7c167584ca9 Mon Sep 17 00:00:00 2001 From: SteveBenz Date: Wed, 19 Jan 2022 16:25:57 -0800 Subject: [PATCH 15/29] Add a bespoke index.lock file to protect fastfetch operations --- GVFS/FastFetch/CheckoutPrefetcher.cs | 139 ++++++++++++---------- GVFS/FastFetch/IndexLock.cs | 108 +++++++++++++++++ GVFS/GVFS.Common/Git/GitIndexGenerator.cs | 12 +- 3 files changed, 197 insertions(+), 62 deletions(-) create mode 100644 GVFS/FastFetch/IndexLock.cs diff --git a/GVFS/FastFetch/CheckoutPrefetcher.cs b/GVFS/FastFetch/CheckoutPrefetcher.cs index 8b85ba794..bfc86560c 100644 --- a/GVFS/FastFetch/CheckoutPrefetcher.cs +++ b/GVFS/FastFetch/CheckoutPrefetcher.cs @@ -72,87 +72,104 @@ public override void Prefetch(string branchOrCommit, bool isBranch) commitToFetch = branchOrCommit; } - this.DownloadMissingCommit(commitToFetch, this.GitObjects); + if (!IndexLock.TryGetLock(this.Enlistment.EnlistmentRoot, this.Tracer, out var indexLock)) + { + this.HasFailures = true; + throw new FetchException("Could not acquire index.lock."); + } - // Configure pipeline - // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper - // Checkout diff output => FindBlobs => BatchDownload => IndexPack => Checkout available blobs - CheckoutStage checkout = new CheckoutStage(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment, this.forceCheckout); - FindBlobsStage blobFinder = new FindBlobsStage(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); - BatchObjectDownloadStage downloader = new BatchObjectDownloadStage(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); - IndexPackStage packIndexer = new IndexPackStage(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); + using (indexLock) + { + this.DownloadMissingCommit(commitToFetch, this.GitObjects); - // Start pipeline - downloader.Start(); - blobFinder.Start(); - checkout.Start(); + // Configure pipeline + // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper + // Checkout diff output => FindBlobs => BatchDownload => IndexPack => Checkout available blobs + CheckoutStage checkout = new CheckoutStage(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment, this.forceCheckout); + FindBlobsStage blobFinder = new FindBlobsStage(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); + BatchObjectDownloadStage downloader = new BatchObjectDownloadStage(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); + IndexPackStage packIndexer = new IndexPackStage(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); - blobFinder.WaitForCompletion(); - this.HasFailures |= blobFinder.HasFailures; + // Start pipeline + downloader.Start(); + blobFinder.Start(); + checkout.Start(); - // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. - packIndexer.Start(); + blobFinder.WaitForCompletion(); + this.HasFailures |= blobFinder.HasFailures; - downloader.WaitForCompletion(); - this.HasFailures |= downloader.HasFailures; + // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. + packIndexer.Start(); - packIndexer.WaitForCompletion(); - this.HasFailures |= packIndexer.HasFailures; + downloader.WaitForCompletion(); + this.HasFailures |= downloader.HasFailures; - // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. - // This prevents availableObjects from completing before packIndexer can push its objects through this link. - checkout.AvailableBlobShas.CompleteAdding(); - checkout.WaitForCompletion(); - this.HasFailures |= checkout.HasFailures; + packIndexer.WaitForCompletion(); + this.HasFailures |= packIndexer.HasFailures; - if (!this.SkipConfigUpdate && !this.HasFailures) - { - this.UpdateRefs(branchOrCommit, isBranch, refs); + // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. + // This prevents availableObjects from completing before packIndexer can push its objects through this link. + checkout.AvailableBlobShas.CompleteAdding(); + checkout.WaitForCompletion(); + this.HasFailures |= checkout.HasFailures; - if (isBranch) + if (!this.SkipConfigUpdate && !this.HasFailures) { - // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist - this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); + this.UpdateRefs(branchOrCommit, isBranch, refs); - using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) + if (isBranch) { - string remoteBranch = refs.GetBranchRefPairs().Single().Key; - GitProcess git = new GitProcess(this.Enlistment); - GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); - if (result.ExitCodeIsFailure) + // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist + this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); + + using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { - activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); - this.HasFailures = true; + string remoteBranch = refs.GetBranchRefPairs().Single().Key; + GitProcess git = new GitProcess(this.Enlistment); + GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); + if (result.ExitCodeIsFailure) + { + activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); + this.HasFailures = true; + } } } - } - bool shouldSignIndex = !this.GetIsIndexSigningOff(); + bool shouldSignIndex = !this.GetIsIndexSigningOff(); - // Update the index - EventMetadata updateIndexMetadata = new EventMetadata(); - updateIndexMetadata.Add("IndexSigningIsOff", shouldSignIndex); - using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata)) - { - Index sourceIndex = this.GetSourceIndex(); - GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldSignIndex); - indexGen.CreateFromHeadTree(indexVersion: 2); - this.HasFailures |= indexGen.HasFailures; - - if (!indexGen.HasFailures) + // Update the index + EventMetadata updateIndexMetadata = new EventMetadata(); + updateIndexMetadata.Add("IndexSigningIsOff", shouldSignIndex); + using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata)) { - Index newIndex = new Index( - this.Enlistment.EnlistmentRoot, - this.Tracer, - Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName), - readOnly: false); - - // Update from disk only if the caller says it is ok via command line - // or if we updated the whole tree and know that all files are up to date - bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; - newIndex.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree, shouldSignIndex, sourceIndex); + Index sourceIndex = this.GetSourceIndex(); + GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldSignIndex); + indexGen.CreateFromHeadTree(indexVersion: 2); + this.HasFailures |= indexGen.HasFailures; + + if (!indexGen.HasFailures) + { + Index newIndex = new Index( + this.Enlistment.EnlistmentRoot, + this.Tracer, + Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName), + readOnly: false); + + // Update from disk only if the caller says it is ok via command line + // or if we updated the whole tree and know that all files are up to date + bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; + newIndex.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree, shouldSignIndex, sourceIndex); + } } } + + indexLock.Release(); + // Note that this releases the lock even if we had failures, and thus even if the index/working-tree + // might need to be repaired. It is up to our caller to monitor the exit code and repair/discard the + // enlistment as-appropriate. If we have an unhandled exception or are killed, the file will be left + // around as well. The thinking here is that if we are killed, it's like our parent is too and thus + // will not be able to take those recovery steps, and leaving the file around ends up being a flag to + // the next command that's run after the machine recovers to sort it out. } } diff --git a/GVFS/FastFetch/IndexLock.cs b/GVFS/FastFetch/IndexLock.cs new file mode 100644 index 000000000..2998c10a6 --- /dev/null +++ b/GVFS/FastFetch/IndexLock.cs @@ -0,0 +1,108 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using System; +using System.IO; + +namespace FastFetch +{ + /// + /// A mechanism for holding the 'index.lock' on a repository for the time it takes to update the index + /// and working tree. + /// + /// + /// + /// This class should not have to exist. If FastFetch was in compliance with the git way of doing + /// business, then would work like this: + /// + /// + /// + /// It would open index.lock like this does - with CreateNew, before it started messing with the working tree. + /// + /// + /// It would have just one class responsible for writing the new index into index.lock (now it has two, + /// and ). And this combined class would write in the + /// file size and timestamp information from the appropriate sources as it goes. + /// + /// + /// It would then reread index.lock (without closing it) and calculate the hash. + /// + /// + /// It would then delete the old index file, close index.lock, and move it to index. + /// + /// + /// + /// This is all in contrast to how it works now, where it has separate operations for updating + /// the working tree, creating an index with no size/timestamp information, and then rewriting + /// it with that information. + /// + /// + /// This class is just a bodge job to make it so that we can leave the code pretty much as-is (and reduce + /// the risk of breaking things) and still get the protection we need against simultaneous git commands + /// being run. + /// + /// + public class IndexLock + : IDisposable + { + private string lockFilePath; + private FileStream lockFileStream; + + private IndexLock(string lockFilePath, FileStream lockFileStream) + { + this.lockFilePath = lockFilePath; + this.lockFileStream = lockFileStream; + } + + /// + /// Attempts to get the index.lock. If it succeeds, it returns an object that must be released + /// when the operation completes with the git repository in good shape. Note that simply disposing + /// the object will not release the lock. + /// + public static bool TryGetLock(string repositoryRoot, ITracer tracer, out IndexLock lockHolder) + { + string lockFilePath = Path.Combine(repositoryRoot, GVFSConstants.DotGit.IndexLock); + try + { + FileStream lockFileStream = File.Open(lockFilePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None); + lockHolder = new IndexLock(lockFilePath, lockFileStream); + return true; + } + catch (Exception ex) + { + tracer.RelatedError("Unable to create: {0}: {1}", lockFilePath, ex.Message); + lockHolder = null; + return false; + } + } + + /// + /// Release the lock, indicating that the index and the working tree are safe for other git operations to use. + /// + public void Release() + { + if (this.lockFilePath == null) + { + return; + } + + if (this.lockFileStream == null) + { + throw new ObjectDisposedException(nameof(IndexLock)); + } + + this.lockFileStream.Dispose(); + this.lockFileStream = null; + + File.Delete(this.lockFilePath); + this.lockFilePath = null; + } + + /// > + public void Dispose() + { + this.lockFileStream?.Dispose(); + this.lockFileStream = null; + } + } +} diff --git a/GVFS/GVFS.Common/Git/GitIndexGenerator.cs b/GVFS/GVFS.Common/Git/GitIndexGenerator.cs index 8f9d7b29c..a79c79c6e 100644 --- a/GVFS/GVFS.Common/Git/GitIndexGenerator.cs +++ b/GVFS/GVFS.Common/Git/GitIndexGenerator.cs @@ -53,7 +53,17 @@ public GitIndexGenerator(ITracer tracer, Enlistment enlistment, bool shouldHashI this.enlistment = enlistment; this.shouldHashIndex = shouldHashIndex; - this.indexLockPath = Path.Combine(enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName + GVFSConstants.DotGit.LockExtension); + // The extension 'lock2' is chosen simply to not be '.lock' because, although this class reasonably + // conforms to how index.lock is supposed to be used, its callers continue to do things to the tree + // and the working tree and even the before this class comes along and after this class has been released. + // FastFetch.IndexLock bodges around this by creating an empty file in the index.lock position, so we + // need to create a different file. See FastFetch.IndexLock for a proposed design to fix this. + // + // Note that there are two callers of this - one is from FastFetch, which we just discussed, and the + // other is from the 'gvfs repair' verb. That environment is special in that it only runs on unmounted + // repo's, so 'index.lock' is irrelevant as a locking mechanism in that context. There can't be git + // commands to lock out. + this.indexLockPath = Path.Combine(enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName + ".lock2"); } public bool HasFailures { get; private set; } From 6a7a14225af06574178c7d5817408bcd238dab01 Mon Sep 17 00:00:00 2001 From: SteveBenz Date: Mon, 24 Jan 2022 13:14:32 -0800 Subject: [PATCH 16/29] Refactor to do ref and index changes at the very end --- GVFS/FastFetch/CheckoutPrefetcher.cs | 63 +++++++++++------------ GVFS/FastFetch/Index.cs | 1 - GVFS/GVFS.Common/Git/GitIndexGenerator.cs | 36 +++++++------ GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs | 2 +- 4 files changed, 52 insertions(+), 50 deletions(-) diff --git a/GVFS/FastFetch/CheckoutPrefetcher.cs b/GVFS/FastFetch/CheckoutPrefetcher.cs index bfc86560c..e87ce97c2 100644 --- a/GVFS/FastFetch/CheckoutPrefetcher.cs +++ b/GVFS/FastFetch/CheckoutPrefetcher.cs @@ -115,36 +115,16 @@ public override void Prefetch(string branchOrCommit, bool isBranch) if (!this.SkipConfigUpdate && !this.HasFailures) { - this.UpdateRefs(branchOrCommit, isBranch, refs); - - if (isBranch) - { - // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist - this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); - - using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) - { - string remoteBranch = refs.GetBranchRefPairs().Single().Key; - GitProcess git = new GitProcess(this.Enlistment); - GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); - if (result.ExitCodeIsFailure) - { - activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); - this.HasFailures = true; - } - } - } - bool shouldSignIndex = !this.GetIsIndexSigningOff(); - // Update the index + // Update the index - note that this will take some time EventMetadata updateIndexMetadata = new EventMetadata(); updateIndexMetadata.Add("IndexSigningIsOff", shouldSignIndex); using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata)) { Index sourceIndex = this.GetSourceIndex(); GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldSignIndex); - indexGen.CreateFromHeadTree(indexVersion: 2); + indexGen.CreateFromRef(commitToFetch, indexVersion: 2, isFinal: false); this.HasFailures |= indexGen.HasFailures; if (!indexGen.HasFailures) @@ -152,13 +132,40 @@ public override void Prefetch(string branchOrCommit, bool isBranch) Index newIndex = new Index( this.Enlistment.EnlistmentRoot, this.Tracer, - Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName), + indexGen.TemporaryIndexFilePath, readOnly: false); // Update from disk only if the caller says it is ok via command line // or if we updated the whole tree and know that all files are up to date bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; newIndex.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree, shouldSignIndex, sourceIndex); + + // All the slow stuff is over, so we will now move the final index into .git\index, shortly followed by + // updating the ref files and releasing index.lock. + string indexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName); + this.Tracer.RelatedEvent(EventLevel.Informational, "MoveUpdatedIndexToFinalLocation", new EventMetadata() { { "UpdatedIndex", indexGen.TemporaryIndexFilePath }, { "Index", indexPath } }); + File.Delete(indexPath); + File.Move(indexGen.TemporaryIndexFilePath, indexPath); + } + } + + this.UpdateRefs(branchOrCommit, isBranch, refs); + + if (isBranch) + { + // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist + this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); + + using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) + { + string remoteBranch = refs.GetBranchRefPairs().Single().Key; + GitProcess git = new GitProcess(this.Enlistment); + GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); + if (result.ExitCodeIsFailure) + { + activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); + this.HasFailures = true; + } } } } @@ -200,18 +207,10 @@ protected override void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs private Index GetSourceIndex() { string indexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName); - string backupIndexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName + ".backup"); if (File.Exists(indexPath)) { - // Note that this moves the current index, leaving nothing behind - // This is intentional as we only need it for the purpose of updating the - // new index and leaving it behind can make updating slower. - this.Tracer.RelatedEvent(EventLevel.Informational, "CreateBackup", new EventMetadata() { { "BackupIndexName", backupIndexPath } }); - File.Delete(backupIndexPath); - File.Move(indexPath, backupIndexPath); - - Index output = new Index(this.Enlistment.EnlistmentRoot, this.Tracer, backupIndexPath, readOnly: true); + Index output = new Index(this.Enlistment.EnlistmentRoot, this.Tracer, indexPath, readOnly: true); output.Parse(); return output; } diff --git a/GVFS/FastFetch/Index.cs b/GVFS/FastFetch/Index.cs index b02c1362f..8beb88963 100644 --- a/GVFS/FastFetch/Index.cs +++ b/GVFS/FastFetch/Index.cs @@ -319,7 +319,6 @@ private void MoveUpdatedIndexToFinalLocation(bool shouldSignIndex) } } - this.tracer.RelatedEvent(EventLevel.Informational, "MoveUpdatedIndexToFinalLocation", new EventMetadata() { { "UpdatedIndex", this.updatedIndexPath }, { "Index", this.indexPath } }); File.Delete(this.indexPath); File.Move(this.updatedIndexPath, this.indexPath); diff --git a/GVFS/GVFS.Common/Git/GitIndexGenerator.cs b/GVFS/GVFS.Common/Git/GitIndexGenerator.cs index a79c79c6e..19c598135 100644 --- a/GVFS/GVFS.Common/Git/GitIndexGenerator.cs +++ b/GVFS/GVFS.Common/Git/GitIndexGenerator.cs @@ -66,18 +66,30 @@ public GitIndexGenerator(ITracer tracer, Enlistment enlistment, bool shouldHashI this.indexLockPath = Path.Combine(enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName + ".lock2"); } + public string TemporaryIndexFilePath => this.indexLockPath; + public bool HasFailures { get; private set; } - public void CreateFromHeadTree(uint indexVersion) + /// Builds an index from scratch based on the current head pointer. + /// The index version see https://git-scm.com/docs/index-format for details on what this means. + /// + /// If true, the index file will be written during this operation. If not, the new index will be + /// left in . + /// + /// + /// The index created by this class has no data from the working tree, so when 'git status' is run, it + /// will calculate the hash of everything in the working tree. + /// + public void CreateFromRef(string refName, uint indexVersion, bool isFinal) { using (ITracer updateIndexActivity = this.tracer.StartActivity("CreateFromHeadTree", EventLevel.Informational)) { - Thread entryWritingThread = new Thread(() => this.WriteAllEntries(indexVersion)); + Thread entryWritingThread = new Thread(() => this.WriteAllEntries(indexVersion, isFinal)); entryWritingThread.Start(); GitProcess git = new GitProcess(this.enlistment); GitProcess.Result result = git.LsTree( - GVFSConstants.DotGit.HeadName, + refName, this.EnqueueEntriesFromLsTree, recursive: true, showAllTrees: false); @@ -102,7 +114,7 @@ private void EnqueueEntriesFromLsTree(string line) } } - private void WriteAllEntries(uint version) + private void WriteAllEntries(uint version, bool isFinal) { try { @@ -127,7 +139,10 @@ private void WriteAllEntries(uint version) } this.AppendIndexSha(); - this.ReplaceExistingIndex(); + if (isFinal) + { + this.ReplaceExistingIndex(); + } } catch (Exception e) { @@ -136,17 +151,6 @@ private void WriteAllEntries(uint version) } } - private string GetDirectoryNameForGitPath(string filename) - { - int idx = filename.LastIndexOf('/'); - if (idx < 0) - { - return "/"; - } - - return filename.Substring(0, idx + 1); - } - private void WriteEntry(BinaryWriter writer, uint version, string sha, string filename, ref uint lastStringLength) { long startPosition = writer.BaseStream.Position; diff --git a/GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs b/GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs index 20dbb6384..153634bad 100644 --- a/GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs +++ b/GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs @@ -46,7 +46,7 @@ public override FixResult TryFixIssues(List messages) } GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldHashIndex: false); - indexGen.CreateFromHeadTree(indexVersion: 4); + indexGen.CreateFromRef(GVFSConstants.DotGit.HeadName, indexVersion: 4, isFinal: true); if (indexGen.HasFailures || this.TryParseIndex(this.indexPath, messages) != IssueType.None) { From a94cf0d246a9328c19db2485440d510531c79ccc Mon Sep 17 00:00:00 2001 From: SteveBenz Date: Mon, 24 Jan 2022 13:34:24 -0800 Subject: [PATCH 17/29] Get rid of the unneed 'backup' index idea - it's always working on a scratch index. --- GVFS/FastFetch/CheckoutPrefetcher.cs | 1 + GVFS/FastFetch/Index.cs | 69 +++++++++------------------- 2 files changed, 23 insertions(+), 47 deletions(-) diff --git a/GVFS/FastFetch/CheckoutPrefetcher.cs b/GVFS/FastFetch/CheckoutPrefetcher.cs index e87ce97c2..5148dfceb 100644 --- a/GVFS/FastFetch/CheckoutPrefetcher.cs +++ b/GVFS/FastFetch/CheckoutPrefetcher.cs @@ -146,6 +146,7 @@ public override void Prefetch(string branchOrCommit, bool isBranch) this.Tracer.RelatedEvent(EventLevel.Informational, "MoveUpdatedIndexToFinalLocation", new EventMetadata() { { "UpdatedIndex", indexGen.TemporaryIndexFilePath }, { "Index", indexPath } }); File.Delete(indexPath); File.Move(indexGen.TemporaryIndexFilePath, indexPath); + newIndex.WriteFastFetchIndexVersionMarker(); } } diff --git a/GVFS/FastFetch/Index.cs b/GVFS/FastFetch/Index.cs index 8beb88963..5704611d3 100644 --- a/GVFS/FastFetch/Index.cs +++ b/GVFS/FastFetch/Index.cs @@ -39,9 +39,7 @@ public class Index private readonly bool readOnly; - // Index paths private readonly string indexPath; - private readonly string updatedIndexPath; private readonly ITracer tracer; private readonly string repoRoot; @@ -63,15 +61,6 @@ public Index( this.indexPath = indexFullPath; this.readOnly = readOnly; - if (this.readOnly) - { - this.updatedIndexPath = this.indexPath; - } - else - { - this.updatedIndexPath = Path.Combine(repoRoot, GVFSConstants.DotGit.Root, UpdatedIndexName); - } - this.versionMarkerFile = Path.Combine(this.repoRoot, GVFSConstants.DotGit.Root, ".fastfetch", "VersionMarker"); } @@ -92,8 +81,8 @@ public Index( /// /// A collection of added or edited files /// Set to true if the working tree is known good and can be used during the update. - /// An optional index to source entry values from - public void UpdateFileSizesAndTimes(BlockingCollection addedOrEditedLocalFiles, bool allowUpdateFromWorkingTree, bool shouldSignIndex, Index backupIndex = null) + /// An optional index to source entry values from + public void UpdateFileSizesAndTimes(BlockingCollection addedOrEditedLocalFiles, bool allowUpdateFromWorkingTree, bool shouldSignIndex, Index sourceIndex = null) { if (this.readOnly) { @@ -102,8 +91,6 @@ public void UpdateFileSizesAndTimes(BlockingCollection addedOrEditedLoca using (ITracer activity = this.tracer.StartActivity("UpdateFileSizesAndTimes", EventLevel.Informational, Keywords.Telemetry, null)) { - File.Copy(this.indexPath, this.updatedIndexPath, overwrite: true); - this.Parse(); bool anyEntriesUpdated = false; @@ -113,13 +100,13 @@ public void UpdateFileSizesAndTimes(BlockingCollection addedOrEditedLoca { // Only populate from the previous index if we believe it's good to populate from // For now, a current FastFetch version marker is the only criteria - if (backupIndex != null) + if (sourceIndex != null) { if (this.IsFastFetchVersionMarkerCurrent()) { using (this.tracer.StartActivity("UpdateFileInformationFromPreviousIndex", EventLevel.Informational, Keywords.Telemetry, null)) { - anyEntriesUpdated |= this.UpdateFileInformationForAllEntries(indexView, backupIndex, allowUpdateFromWorkingTree); + anyEntriesUpdated |= this.UpdateFileInformationForAllEntries(indexView, sourceIndex, allowUpdateFromWorkingTree); } if (addedOrEditedLocalFiles != null) @@ -139,22 +126,18 @@ public void UpdateFileSizesAndTimes(BlockingCollection addedOrEditedLoca indexView.Flush(); } - if (anyEntriesUpdated) + if (shouldSignIndex) { - this.MoveUpdatedIndexToFinalLocation(shouldSignIndex); - } - else - { - File.Delete(this.updatedIndexPath); + this.SignIndex(); } } } public void Parse() { - using (ITracer activity = this.tracer.StartActivity("ParseIndex", EventLevel.Informational, Keywords.Telemetry, new EventMetadata() { { "Index", this.updatedIndexPath } })) + using (ITracer activity = this.tracer.StartActivity("ParseIndex", EventLevel.Informational, Keywords.Telemetry, new EventMetadata() { { "Index", this.indexPath } })) { - using (Stream indexStream = new FileStream(this.updatedIndexPath, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (Stream indexStream = new FileStream(this.indexPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { this.ParseIndex(indexStream); } @@ -173,7 +156,7 @@ private static string FromGitRelativePathToDotnetFullPath(string path, string re private MemoryMappedFile GetMemoryMappedFile() { - return MemoryMappedFile.CreateFromFile(this.updatedIndexPath, FileMode.Open); + return MemoryMappedFile.CreateFromFile(this.indexPath, FileMode.Open); } private bool UpdateFileInformationFromWorkingTree(MemoryMappedViewAccessor indexView) @@ -296,36 +279,28 @@ private bool UpdateFileInformationForAllEntries(MemoryMappedViewAccessor indexVi return (updatedEntriesFromOtherIndex > 0) || (updatedEntriesFromDisk > 0); } - private void MoveUpdatedIndexToFinalLocation(bool shouldSignIndex) + private void SignIndex() { - if (shouldSignIndex) + using (ITracer activity = this.tracer.StartActivity("SignIndex", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { - using (ITracer activity = this.tracer.StartActivity("SignIndex", EventLevel.Informational, Keywords.Telemetry, metadata: null)) + using (FileStream fs = File.Open(this.indexPath, FileMode.Open, FileAccess.ReadWrite)) { - using (FileStream fs = File.Open(this.updatedIndexPath, FileMode.Open, FileAccess.ReadWrite)) + // Truncate the old hash off. The Index class is expected to preserve any existing hash. + fs.SetLength(fs.Length - 20); + using (HashingStream hashStream = new HashingStream(fs)) { - // Truncate the old hash off. The Index class is expected to preserve any existing hash. - fs.SetLength(fs.Length - 20); - using (HashingStream hashStream = new HashingStream(fs)) - { - fs.Position = 0; - hashStream.CopyTo(Stream.Null); - byte[] hash = hashStream.Hash; + fs.Position = 0; + hashStream.CopyTo(Stream.Null); + byte[] hash = hashStream.Hash; - // The fs pointer is now where the old hash used to be. Perfect. :) - fs.Write(hash, 0, hash.Length); - } + // The fs pointer is now where the old hash used to be. Perfect. :) + fs.Write(hash, 0, hash.Length); } } } - - File.Delete(this.indexPath); - File.Move(this.updatedIndexPath, this.indexPath); - - this.WriteFastFetchIndexVersionMarker(); } - private void WriteFastFetchIndexVersionMarker() + public void WriteFastFetchIndexVersionMarker() { if (File.Exists(this.versionMarkerFile)) { @@ -374,7 +349,7 @@ private void ParseIndex(Stream indexStream) this.entryCount = this.ReadUInt32(buffer, indexStream); - this.tracer.RelatedEvent(EventLevel.Informational, "IndexData", new EventMetadata() { { "Index", this.updatedIndexPath }, { "Version", this.IndexVersion }, { "entryCount", this.entryCount } }, Keywords.Telemetry); + this.tracer.RelatedEvent(EventLevel.Informational, "IndexData", new EventMetadata() { { "Index", this.indexPath }, { "Version", this.IndexVersion }, { "entryCount", this.entryCount } }, Keywords.Telemetry); this.indexEntryOffsets = new Dictionary((int)this.entryCount, GVFSPlatform.Instance.Constants.PathComparer); From 8f1cdbe59f7cefa4d85fb0111946658947cbca51 Mon Sep 17 00:00:00 2001 From: SteveBenz Date: Mon, 24 Jan 2022 15:44:43 -0800 Subject: [PATCH 18/29] Make it so that the index.lock file is just a lock file, not attempting to pass on pass-fail information to future instances --- GVFS/FastFetch/CheckoutPrefetcher.cs | 45 +++++++++++----------------- GVFS/FastFetch/IndexLock.cs | 39 ++++++------------------ 2 files changed, 26 insertions(+), 58 deletions(-) diff --git a/GVFS/FastFetch/CheckoutPrefetcher.cs b/GVFS/FastFetch/CheckoutPrefetcher.cs index 5148dfceb..e8a48a941 100644 --- a/GVFS/FastFetch/CheckoutPrefetcher.cs +++ b/GVFS/FastFetch/CheckoutPrefetcher.cs @@ -72,13 +72,7 @@ public override void Prefetch(string branchOrCommit, bool isBranch) commitToFetch = branchOrCommit; } - if (!IndexLock.TryGetLock(this.Enlistment.EnlistmentRoot, this.Tracer, out var indexLock)) - { - this.HasFailures = true; - throw new FetchException("Could not acquire index.lock."); - } - - using (indexLock) + using (new IndexLock(this.Enlistment.EnlistmentRoot, this.Tracer)) { this.DownloadMissingCommit(commitToFetch, this.GitObjects); @@ -142,7 +136,7 @@ public override void Prefetch(string branchOrCommit, bool isBranch) // All the slow stuff is over, so we will now move the final index into .git\index, shortly followed by // updating the ref files and releasing index.lock. - string indexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName); + string indexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName); this.Tracer.RelatedEvent(EventLevel.Informational, "MoveUpdatedIndexToFinalLocation", new EventMetadata() { { "UpdatedIndex", indexGen.TemporaryIndexFilePath }, { "Index", indexPath } }); File.Delete(indexPath); File.Move(indexGen.TemporaryIndexFilePath, indexPath); @@ -150,34 +144,29 @@ public override void Prefetch(string branchOrCommit, bool isBranch) } } - this.UpdateRefs(branchOrCommit, isBranch, refs); - - if (isBranch) + if (!this.HasFailures) { - // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist - this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); + this.UpdateRefs(branchOrCommit, isBranch, refs); - using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) + if (isBranch) { - string remoteBranch = refs.GetBranchRefPairs().Single().Key; - GitProcess git = new GitProcess(this.Enlistment); - GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); - if (result.ExitCodeIsFailure) + // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist + this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); + + using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { - activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); - this.HasFailures = true; + string remoteBranch = refs.GetBranchRefPairs().Single().Key; + GitProcess git = new GitProcess(this.Enlistment); + GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); + if (result.ExitCodeIsFailure) + { + activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); + this.HasFailures = true; + } } } } } - - indexLock.Release(); - // Note that this releases the lock even if we had failures, and thus even if the index/working-tree - // might need to be repaired. It is up to our caller to monitor the exit code and repair/discard the - // enlistment as-appropriate. If we have an unhandled exception or are killed, the file will be left - // around as well. The thinking here is that if we are killed, it's like our parent is too and thus - // will not be able to take those recovery steps, and leaving the file around ends up being a flag to - // the next command that's run after the machine recovers to sort it out. } } diff --git a/GVFS/FastFetch/IndexLock.cs b/GVFS/FastFetch/IndexLock.cs index 2998c10a6..aee87733a 100644 --- a/GVFS/FastFetch/IndexLock.cs +++ b/GVFS/FastFetch/IndexLock.cs @@ -1,5 +1,6 @@ using GVFS.Common; using GVFS.Common.Git; +using GVFS.Common.Prefetch; using GVFS.Common.Tracing; using System; using System.IO; @@ -8,7 +9,8 @@ namespace FastFetch { /// /// A mechanism for holding the 'index.lock' on a repository for the time it takes to update the index - /// and working tree. + /// and working tree. It attempts to create the file in the constructor and throws if that fails. + /// It closes and deletes index.lock on dispose. /// /// /// @@ -48,38 +50,22 @@ public class IndexLock private string lockFilePath; private FileStream lockFileStream; - private IndexLock(string lockFilePath, FileStream lockFileStream) + public IndexLock(string repositoryRoot, ITracer tracer) { - this.lockFilePath = lockFilePath; - this.lockFileStream = lockFileStream; - } - - /// - /// Attempts to get the index.lock. If it succeeds, it returns an object that must be released - /// when the operation completes with the git repository in good shape. Note that simply disposing - /// the object will not release the lock. - /// - public static bool TryGetLock(string repositoryRoot, ITracer tracer, out IndexLock lockHolder) - { - string lockFilePath = Path.Combine(repositoryRoot, GVFSConstants.DotGit.IndexLock); + this.lockFilePath = Path.Combine(repositoryRoot, GVFSConstants.DotGit.IndexLock); try { - FileStream lockFileStream = File.Open(lockFilePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None); - lockHolder = new IndexLock(lockFilePath, lockFileStream); - return true; + this.lockFileStream = File.Open(lockFilePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None); } catch (Exception ex) { tracer.RelatedError("Unable to create: {0}: {1}", lockFilePath, ex.Message); - lockHolder = null; - return false; + throw new BlobPrefetcher.FetchException("Could not acquire index.lock."); } } - /// - /// Release the lock, indicating that the index and the working tree are safe for other git operations to use. - /// - public void Release() + /// > + public void Dispose() { if (this.lockFilePath == null) { @@ -97,12 +83,5 @@ public void Release() File.Delete(this.lockFilePath); this.lockFilePath = null; } - - /// > - public void Dispose() - { - this.lockFileStream?.Dispose(); - this.lockFileStream = null; - } } } From 52b7d44de715746e4527aec7111d809351e8aa1d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 31 Jan 2022 11:00:18 -0500 Subject: [PATCH 19/29] *.csproj: update MicroBuild package names Necessary for new singing build changes. --- GVFS/FastFetch/FastFetch.csproj | 2 +- GVFS/GVFS.Installers/GVFS.Installers.csproj | 2 +- GVFS/GVFS.Payload/GVFS.Payload.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/GVFS/FastFetch/FastFetch.csproj b/GVFS/FastFetch/FastFetch.csproj index 52bb94654..63f5736ed 100644 --- a/GVFS/FastFetch/FastFetch.csproj +++ b/GVFS/FastFetch/FastFetch.csproj @@ -13,7 +13,7 @@ - + diff --git a/GVFS/GVFS.Installers/GVFS.Installers.csproj b/GVFS/GVFS.Installers/GVFS.Installers.csproj index 1f5b692d1..bc83b9728 100644 --- a/GVFS/GVFS.Installers/GVFS.Installers.csproj +++ b/GVFS/GVFS.Installers/GVFS.Installers.csproj @@ -15,7 +15,7 @@ - + diff --git a/GVFS/GVFS.Payload/GVFS.Payload.csproj b/GVFS/GVFS.Payload/GVFS.Payload.csproj index 1df5a3210..18b650de7 100644 --- a/GVFS/GVFS.Payload/GVFS.Payload.csproj +++ b/GVFS/GVFS.Payload/GVFS.Payload.csproj @@ -13,7 +13,7 @@ - + From 7587d284579aee10e579b931bfd7297272c88b6f Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 31 Jan 2022 12:15:00 -0500 Subject: [PATCH 20/29] Revert "Merge pull request #1765 from derrickstolee/signing" This reverts commit 183f1965c0dda4b73aedfccc9f17a3ba2c0fd7c2, reversing changes made to 2cd3558ab4d5bb1f7e42d55dd07df43889eba47a. --- GVFS/FastFetch/FastFetch.csproj | 2 +- GVFS/GVFS.Installers/GVFS.Installers.csproj | 2 +- GVFS/GVFS.Payload/GVFS.Payload.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/GVFS/FastFetch/FastFetch.csproj b/GVFS/FastFetch/FastFetch.csproj index 63f5736ed..52bb94654 100644 --- a/GVFS/FastFetch/FastFetch.csproj +++ b/GVFS/FastFetch/FastFetch.csproj @@ -13,7 +13,7 @@ - + diff --git a/GVFS/GVFS.Installers/GVFS.Installers.csproj b/GVFS/GVFS.Installers/GVFS.Installers.csproj index bc83b9728..1f5b692d1 100644 --- a/GVFS/GVFS.Installers/GVFS.Installers.csproj +++ b/GVFS/GVFS.Installers/GVFS.Installers.csproj @@ -15,7 +15,7 @@ - + diff --git a/GVFS/GVFS.Payload/GVFS.Payload.csproj b/GVFS/GVFS.Payload/GVFS.Payload.csproj index 18b650de7..1df5a3210 100644 --- a/GVFS/GVFS.Payload/GVFS.Payload.csproj +++ b/GVFS/GVFS.Payload/GVFS.Payload.csproj @@ -13,7 +13,7 @@ - + From bc2f4cff9e940a66c3ed45d05db4fb5084e0035b Mon Sep 17 00:00:00 2001 From: Git Fundamentals Date: Tue, 1 Feb 2022 12:34:00 -0500 Subject: [PATCH 21/29] Update Readme --- Readme.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Readme.md b/Readme.md index b0578ff4f..2ee0517b9 100644 --- a/Readme.md +++ b/Readme.md @@ -1,5 +1,7 @@ # VFS for Git +**Notice:** With the release of VFS for Git 2.32, VFS for Git is in maintenance mode. Only required updates as a reaction to critical security vulnerabilities will prompt a release. + |Branch|Unit Tests|Functional Tests|Large Repo Perf|Large Repo Build| |:--:|:--:|:--:|:--:|:--:| |**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=master)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=master)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=master)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7179&branchName=master)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=master)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7180&branchName=master)| From d7ec528015d0b1434cc338ec26b7d0b2db81b5df Mon Sep 17 00:00:00 2001 From: "microsoft-github-policy-service[bot]" <77245923+microsoft-github-policy-service[bot]@users.noreply.github.com> Date: Wed, 31 Aug 2022 15:22:05 +0000 Subject: [PATCH 22/29] Microsoft mandatory file --- SECURITY.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..869fdfe2b --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). + + From fdfe4d47134428c82886c52c3e84a7d8c0485182 Mon Sep 17 00:00:00 2001 From: Lessley Dennington Date: Thu, 13 Oct 2022 12:01:52 -0700 Subject: [PATCH 23/29] gvfs common: remove reference to Nuget.Commands The Nuget.Commmands package version 4.9.2 contains a known vulnerability. Fortunately we are no longer using this package and can mitigate by removing our reference to it. --- GVFS/GVFS.Common/GVFS.Common.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/GVFS/GVFS.Common/GVFS.Common.csproj b/GVFS/GVFS.Common/GVFS.Common.csproj index 9cc9a5cb8..1c4dd282b 100644 --- a/GVFS/GVFS.Common/GVFS.Common.csproj +++ b/GVFS/GVFS.Common/GVFS.Common.csproj @@ -10,7 +10,6 @@ - From e8ebc5e36f982537cecc9f16584656e1fafe679c Mon Sep 17 00:00:00 2001 From: Lessley Dennington Date: Thu, 13 Oct 2022 14:23:13 -0700 Subject: [PATCH 24/29] dependencies: upgrade newtonsoft.json Update newtonsoft.json to 13.0.1 to mitigate exposure to security vulnerability associated with previous versions. --- GVFS/GVFS.Common/GVFS.Common.csproj | 2 +- GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj | 2 +- GVFS/GVFS.GVFlt/GVFS.GVFlt.csproj | 2 +- GVFS/GVFS.Service/GVFS.Service.csproj | 2 +- GVFS/GVFS.Virtualization/GVFS.Virtualization.csproj | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.Common/GVFS.Common.csproj b/GVFS/GVFS.Common/GVFS.Common.csproj index 1c4dd282b..e148b1926 100644 --- a/GVFS/GVFS.Common/GVFS.Common.csproj +++ b/GVFS/GVFS.Common/GVFS.Common.csproj @@ -8,7 +8,7 @@ - + diff --git a/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj b/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj index 345a70bcb..1da3886d4 100644 --- a/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj +++ b/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj @@ -10,7 +10,7 @@ - + diff --git a/GVFS/GVFS.GVFlt/GVFS.GVFlt.csproj b/GVFS/GVFS.GVFlt/GVFS.GVFlt.csproj index 16074079a..a436520f9 100644 --- a/GVFS/GVFS.GVFlt/GVFS.GVFlt.csproj +++ b/GVFS/GVFS.GVFlt/GVFS.GVFlt.csproj @@ -5,7 +5,7 @@ - + diff --git a/GVFS/GVFS.Service/GVFS.Service.csproj b/GVFS/GVFS.Service/GVFS.Service.csproj index 99fa85b3e..6d49897ce 100644 --- a/GVFS/GVFS.Service/GVFS.Service.csproj +++ b/GVFS/GVFS.Service/GVFS.Service.csproj @@ -12,7 +12,7 @@ - + diff --git a/GVFS/GVFS.Virtualization/GVFS.Virtualization.csproj b/GVFS/GVFS.Virtualization/GVFS.Virtualization.csproj index 0097850b8..bc5fa77f8 100644 --- a/GVFS/GVFS.Virtualization/GVFS.Virtualization.csproj +++ b/GVFS/GVFS.Virtualization/GVFS.Virtualization.csproj @@ -11,7 +11,7 @@ - + From 3514124740c7212f48882c2614d25a485d999dc9 Mon Sep 17 00:00:00 2001 From: Lessley Dennington Date: Fri, 14 Oct 2022 07:12:47 -0700 Subject: [PATCH 25/29] dependencies: upgrade SharpZipLib Update newtonsoft.json to 13.0.1 to mitigate exposure to security vulnerability associated with previous versions. --- GVFS/GVFS.Common/GVFS.Common.csproj | 8 +++----- GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj | 2 +- GVFS/GVFS.Service/GVFS.Service.csproj | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.Common/GVFS.Common.csproj b/GVFS/GVFS.Common/GVFS.Common.csproj index e148b1926..e96c88ba7 100644 --- a/GVFS/GVFS.Common/GVFS.Common.csproj +++ b/GVFS/GVFS.Common/GVFS.Common.csproj @@ -9,7 +9,7 @@ - + @@ -18,13 +18,11 @@ - + - + diff --git a/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj b/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj index 1da3886d4..0a64e7440 100644 --- a/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj +++ b/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj @@ -11,7 +11,7 @@ - + diff --git a/GVFS/GVFS.Service/GVFS.Service.csproj b/GVFS/GVFS.Service/GVFS.Service.csproj index 6d49897ce..821141326 100644 --- a/GVFS/GVFS.Service/GVFS.Service.csproj +++ b/GVFS/GVFS.Service/GVFS.Service.csproj @@ -13,7 +13,7 @@ - + From 797d7f3c9e2881a44e1544b3d4ca97e8cc8a67b0 Mon Sep 17 00:00:00 2001 From: Neeraj Singh Date: Wed, 17 Feb 2021 14:43:12 -0800 Subject: [PATCH 26/29] index-parser: Do not use DateTime.UtcNow for controlling logging. UtcNow spends a lot of time getting a very precise system time, including leap seconds and interpolation through QueryPerformanceCounter. Use Environment.TickCount (milliseconds since boot) instead, since that just retrieves a value in shared memory that the kernel keeps up to date. This value is still suitable for controlling how much log output index parsing produces. This UtcNow call consumes 15% of the time under GitIndexParser.ParseIndex. See https://github.com/dotnet/runtime/blob/master/src/libraries/System.Private.CoreLib/src/System/DateTime.Windows.cs. --- .../Projection/GitIndexProjection.GitIndexParser.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 20c451cbc..c05348e2b 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -238,8 +238,8 @@ private FileSystemTaskResult ParseIndex( uint entryCount = this.ReadFromIndexHeader(); // Don't want to flood the logs on large indexes so only log every 500ms - const int LoggingTicksThreshold = 5000000; - long nextLogTicks = DateTime.UtcNow.Ticks + LoggingTicksThreshold; + const int LoggingTicksThreshold = 500; + int nextLogTicks = Environment.TickCount + LoggingTicksThreshold; SortedFolderEntries.InitializePools(tracer, entryCount); LazyUTF8String.InitializePools(tracer, entryCount); @@ -329,10 +329,11 @@ private FileSystemTaskResult ParseIndex( return result; } - if (DateTime.UtcNow.Ticks > nextLogTicks) + int curTicks = Environment.TickCount; + if (curTicks - nextLogTicks > 0) { tracer.RelatedInfo($"{i}/{entryCount} index entries parsed."); - nextLogTicks = DateTime.UtcNow.Ticks + LoggingTicksThreshold; + nextLogTicks = curTicks + LoggingTicksThreshold; } } From cf6b45f355cdb6c936b75339b4086e34742211f7 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Fri, 26 Jan 2024 13:34:47 -0800 Subject: [PATCH 27/29] fix: #1802 confirm cached credential before rejecting --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 27 ++++++++++++++++-- .../Git/GitAuthenticationTests.cs | 28 +++++++++++++++++++ .../GVFS.UnitTests/Mock/Git/MockGitProcess.cs | 5 ++-- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index bed156468..970a71879 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -91,9 +91,27 @@ public void RejectCredentials(ITracer tracer, string credentialString) { lock (this.gitAuthLock) { + var cachedCredentialAtStartOfReject = this.cachedCredentialString; // Don't stomp a different credential - if (credentialString == this.cachedCredentialString && this.cachedCredentialString != null) + if (credentialString == cachedCredentialAtStartOfReject && cachedCredentialAtStartOfReject != null) { + // We can't assume that the credential store's cached credential is the same as the one we have. + // Reload the credential from the store to ensure we're rejecting the correct one. + var attemptsBeforeCheckingExistingCredential = this.numberOfAttempts; + if (this.TryCallGitCredential(tracer, out string getCredentialError)) + { + if (this.cachedCredentialString != cachedCredentialAtStartOfReject) + { + // If the store already had a different credential, we don't want to reject it without trying it. + this.isCachedCredentialStringApproved = false; + return; + } + } + else + { + tracer.RelatedWarning(getCredentialError); + } + // If we can we should pass the actual username/password values we used (and found to be invalid) // to `git-credential reject` so the credential helpers can attempt to check if they're erasing // the expected credentials, if they so choose to. @@ -121,7 +139,12 @@ public void RejectCredentials(ITracer tracer, string credentialString) this.cachedCredentialString = null; this.isCachedCredentialStringApproved = false; - this.UpdateBackoff(); + + // Backoff may have already been incremented by a failure in TryCallGitCredential + if (attemptsBeforeCheckingExistingCredential == this.numberOfAttempts) + { + this.UpdateBackoff(); + } } } } diff --git a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs index 648d6c463..c083ac586 100644 --- a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs +++ b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs @@ -1,3 +1,4 @@ +using System; using System.Linq; using GVFS.Common.Git; using GVFS.Tests; @@ -245,6 +246,33 @@ public void DontStoreDifferentCredentialFromCachedValue() gitProcess.StoredCredentials.Single().Key.ShouldEqual("mock://repoUrl"); } + [TestCase] + public void RejectionShouldNotBeSentIfUnderlyingTokenHasChanged() + { + MockTracer tracer = new MockTracer(); + MockGitProcess gitProcess = this.GetGitProcess(); + + GitAuthentication dut = new GitAuthentication(gitProcess, "mock://repoUrl"); + dut.TryInitializeAndRequireAuth(tracer, out _); + + // Get and store an initial value that will be cached + string authString; + dut.TryGetCredentials(tracer, out authString, out _).ShouldBeTrue(); + dut.ApproveCredentials(tracer, authString); + + // Change the underlying token + gitProcess.SetExpectedCommandResult( + $"{AzureDevOpsUseHttpPathString} credential fill", + () => new GitProcess.Result("username=username\r\npassword=password" + Guid.NewGuid() + "\r\n", string.Empty, GitProcess.Result.SuccessCode)); + + // Try and reject it. We should get a new token, but without forwarding the rejection to the + // underlying credential store + dut.RejectCredentials(tracer, authString); + dut.TryGetCredentials(tracer, out var newAuthString, out _).ShouldBeTrue(); + newAuthString.ShouldNotEqual(authString); + gitProcess.CredentialRejections.ShouldBeEmpty(); + } + private MockGitProcess GetGitProcess() { MockGitProcess gitProcess = new MockGitProcess(); diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs index 26923e180..29f2651ba 100644 --- a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; namespace GVFS.UnitTests.Mock.Git @@ -91,7 +92,7 @@ protected override Result InvokeGitImpl( return new Result(string.Empty, string.Empty, Result.GenericFailureCode); } - Predicate commandMatchFunction = + Func commandMatchFunction = (CommandInfo commandInfo) => { if (commandInfo.MatchPrefix) @@ -104,7 +105,7 @@ protected override Result InvokeGitImpl( } }; - CommandInfo matchedCommand = this.expectedCommandInfos.Find(commandMatchFunction); + CommandInfo matchedCommand = this.expectedCommandInfos.Last(commandMatchFunction); matchedCommand.ShouldNotBeNull("Unexpected command: " + command); return matchedCommand.Result(); From 90cf37d120c3ec311e4cd7ebe5471d38560e21a3 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Thu, 15 Feb 2024 08:51:42 -0800 Subject: [PATCH 28/29] Style fixes from pull request feeback --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index 970a71879..d82053ef5 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -91,13 +91,13 @@ public void RejectCredentials(ITracer tracer, string credentialString) { lock (this.gitAuthLock) { - var cachedCredentialAtStartOfReject = this.cachedCredentialString; + string cachedCredentialAtStartOfReject = this.cachedCredentialString; // Don't stomp a different credential if (credentialString == cachedCredentialAtStartOfReject && cachedCredentialAtStartOfReject != null) { // We can't assume that the credential store's cached credential is the same as the one we have. // Reload the credential from the store to ensure we're rejecting the correct one. - var attemptsBeforeCheckingExistingCredential = this.numberOfAttempts; + int attemptsBeforeCheckingExistingCredential = this.numberOfAttempts; if (this.TryCallGitCredential(tracer, out string getCredentialError)) { if (this.cachedCredentialString != cachedCredentialAtStartOfReject) From cbd0cefb9cc87744286fde23632860354f5c62ee Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 15 Feb 2024 10:01:27 -0500 Subject: [PATCH 29/29] GVFSVerb: Use OAuth credentials by default To more rapidly adopt OAuth tokens, set that as the default for GVFS repos. This will update on mount, so all users will update to this mode when they upgrade to a version including this commit. This may cause some initial frustration as the first time I ran a fetch in OAuth mode my local clone had a failure with GCM and defaulted to username/password checks over command line. The second fetch worked just fine, though. --- GVFS/GVFS/CommandLine/GVFSVerb.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 8449b5bab..cd74600e4 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -312,6 +312,11 @@ public static bool TrySetRequiredGitConfigSettings(Enlistment enlistment) // Disable the builtin FS Monitor in case it was enabled globally. { "core.useBuiltinFSMonitor", "false" }, + + // Set the GCM credential method to use OAuth tokens. + // See https://github.com/git-ecosystem/git-credential-manager/blob/release/docs/configuration.md#credentialazreposcredentialtype + // for more information. + { "credential.azreposCredentialType", "oauth" }, }; if (!TrySetConfig(enlistment, requiredSettings, isRequired: true))