diff --git a/eng/common/scripts/Helpers/git-helpers.ps1 b/eng/common/scripts/Helpers/git-helpers.ps1 index e9feafe4636f..2365304bedcb 100644 --- a/eng/common/scripts/Helpers/git-helpers.ps1 +++ b/eng/common/scripts/Helpers/git-helpers.ps1 @@ -36,3 +36,75 @@ function Get-ChangedFiles { } return $changedFiles } + +class ConflictedFile { + [string]$LeftSource = "" + [string]$RightSource = "" + [string]$Content = "" + [string]$Path = "" + [boolean]$IsConflicted = $false + + ConflictedFile([string]$File = "") { + if (!(Test-Path $File)) { + throw "File $File does not exist, pass a valid file path to the constructor." + } + + # Normally we would use Resolve-Path $file, but git only can handle relative paths using git show : + # Therefore, just maintain whatever the path is given to us. Left() and Right() should therefore be called from the same + # directory as where we defined the relative path to the target file. + $this.Path = $File + $this.Content = Get-Content -Raw $File + + $this.ParseContent($this.Content) + } + + [array] Left(){ + if ($this.IsConflicted) { + # we are forced to get this line by line and reassemble via join because of how powershell is interacting with + # git show --textconv commitsh:path + # powershell ignores the newlines with and without --textconv, which results in a json file without original spacing. + # by forcefully reading into the array line by line, the whitespace is preserved. we're relying on gits autoconverstion of clrf to lf + # to ensure that the line endings are consistent. + Write-Host "git show $($this.LeftSource):$($this.Path)" + $tempContent = git show ("$($this.LeftSource):$($this.Path)") + return $tempContent -split "`r?`n" + } + else { + return $this.Content + } + } + + [array] Right(){ + if ($this.IsConflicted) { + Write-Host "git show $($this.RightSource):$($this.Path)" + $tempContent = git show ("$($this.RightSource):$($this.Path)") + return $tempContent -split "`r?`n" + } + else { + return $this.Content + } + } + + [void] ParseContent([string]$IncomingContent) { + $lines = $IncomingContent -split "`r?`n" + $l = @() + $r = @() + + foreach($line in $lines) { + if ($line -match "^<<<<<<<\s*(.+)") { + $this.IsConflicted = $true + $this.LeftSource = $matches[1] + continue + } + elseif ($line -match "^>>>>>>>\s*(.+)") { + $this.IsConflicted = $true + $this.RightSource = $matches[1] + continue + } + + if ($this.LeftSource -and $this.RightSource) { + break + } + } + } +} diff --git a/eng/common/scripts/Helpers/git-helpers.tests.ps1 b/eng/common/scripts/Helpers/git-helpers.tests.ps1 new file mode 100644 index 000000000000..f048fe8946a4 --- /dev/null +++ b/eng/common/scripts/Helpers/git-helpers.tests.ps1 @@ -0,0 +1,57 @@ +# Install-Module -Name Pester -Force -SkipPublisherCheck +# Invoke-Pester -Passthru path/to/git-helpers.tests.ps1 +BeforeAll { + . $PSScriptRoot/git-helpers.ps1 + + $RunFolder = "$PSScriptRoot/.testruns" + + if (Test-Path $RunFolder){ + Remove-Item -Recurse -Force $RunFolder + } + + New-Item -ItemType Directory -Path $RunFolder +} + +Describe "git-helpers.ps1 tests"{ + Context "Test Parse-ConflictedFile" { + It "Parses a basic conflicted file" { + $content = @' +{ + "AssetsRepo": "Azure/azure-sdk-assets-integration", + "AssetsRepoPrefixPath": "python", + "TagPrefix": "python/storage/azure-storage-blob", +<<<<<<< HEAD + "Tag": "integration/example/storage_feature_addition2" +======= + "Tag": "integration/example/storage_feature_addition1" +>>>>>>> test-storage-tag-combination +} +'@ + $contentPath = Join-Path $RunFolder "basic_conflict_test.json" + Set-Content -Path $contentPath -Value $content + + $resolution = [ConflictedFile]::new($contentPath) + $resolution.IsConflicted | Should -Be $true + $resolution.LeftSource | Should -Be "HEAD" + $resolution.RightSource | Should -Be "test-storage-tag-combination" + } + + It "Recognizes when no conflicts are present" { + $content = @' +{ + "AssetsRepo": "Azure/azure-sdk-assets-integration", + "AssetsRepoPrefixPath": "python", + "TagPrefix": "python/storage/azure-storage-blob", + "Tag": "integration/example/storage_feature_addition1" +} +'@ + $contentPath = Join-Path $RunFolder "no_conflict_test.json" + Set-Content -Path $contentPath -Value $content + + $resolution = [ConflictedFile]::new($contentPath) + $resolution.IsConflicted | Should -Be $false + $resolution.LeftSource | Should -Be "" + $resolution.RightSource | Should -Be "" + } + } +} \ No newline at end of file diff --git a/eng/common/testproxy/onboarding/common-asset-functions.ps1 b/eng/common/testproxy/onboarding/common-asset-functions.ps1 index 3caa6654c133..3d7bcf605584 100644 --- a/eng/common/testproxy/onboarding/common-asset-functions.ps1 +++ b/eng/common/testproxy/onboarding/common-asset-functions.ps1 @@ -57,6 +57,24 @@ class Version { } } +Function Resolve-Proxy { + $testProxyExe = "test-proxy" + # this script requires the presence of the test-proxy on the PATH + $proxyToolPresent = Test-Exe-In-Path -ExeToLookFor "test-proxy" -ExitOnError $false + $proxyStandalonePresent = Test-Exe-In-Path -ExeToLookFor "Azure.Sdk.Tools.TestProxy" -ExitOnError $false + + if (-not $proxyToolPresent -and -not $proxyStandalonePresent) { + Write-Error "This script requires the presence of a test-proxy executable to complete its operations. Exiting." + exit 1 + } + + if (-not $proxyToolPresent) { + $testProxyExe = "Azure.Sdk.Tools.TestProxy" + } + + return $testProxyExe +} + Function Test-Exe-In-Path { Param([string] $ExeToLookFor, [bool]$ExitOnError = $true) if ($null -eq (Get-Command $ExeToLookFor -ErrorAction SilentlyContinue)) { diff --git a/eng/common/testproxy/scripts/resolve-asset-conflict/README.md b/eng/common/testproxy/scripts/resolve-asset-conflict/README.md new file mode 100644 index 000000000000..13326596e32e --- /dev/null +++ b/eng/common/testproxy/scripts/resolve-asset-conflict/README.md @@ -0,0 +1,62 @@ +# Merge Proxy Tags Script + +This script is intended to allow easy resolution of a conflicting `assets.json` file. + +In most cases where two branches `X` and `Y` have progressed alongside each other, a simple + +`git checkout X && git merge Y` can successfully merge _other_ than the `assets.json` file. + +That often will end up looking like this: + +```text +{ + "AssetsRepo": "Azure/azure-sdk-assets-integration", + "AssetsRepoPrefixPath": "python", + "TagPrefix": "python/storage/azure-storage-blob", +<<<<<<< HEAD + "Tag": "integration/example/storage_feature_addition2" +======= + "Tag": "integration/example/storage_feature_addition1" +>>>>>>> test-storage-tag-combination +} +``` + +This script uses `git` to tease out the source and target tags, then merge the incoming tag into the recordings of the base tag. + +This script should _only_ be used on an already conflicted `assets.json` file. Otherwise, no action will be executed. + +## Usage + +### PreReqs + +- Must have []`pshell 6+`](https://learn.microsoft.com/powershell/scripting/install/installing-powershell-on-windows) +- Must have `git` available on your PATH +- Must have the `test-proxy` available on your PATH + - `test-proxy` is honored when the proxy is installed as a `dotnet tool` + - `Azure.Sdk.Tools.TestProxy` is honored when the standalone executable is on your PATH + - Defaults to `dotnet tool` if both are present on the PATH. + +### Call the script + +For simplicity when resolving merge-conflicts, invoke the script from the root of the repo. The help instructions from `merge-asset-tags` use paths relative from repo root. + +```powershell +# including context to get into a merge conflict +cd "path/to/language/repo/root" +git checkout base-branch +git merge target-branch +# auto resolve / merge conflicting tag values +./eng/common/testproxy/scripts/resolve-asset-conflict/resolve-asset-conflict.ps1 sdk/storage/azure-storage-blob/assets.json +# user pushes +test-proxy push -a sdk/storage/azure-storage-blob/assets.json +``` + +### Resolving conflicts + +When an assets.json merge has conflicts on the **test recordings** side, the `merge-proxy-tags` script will exit with an error describing how to re-invoke the `merge-proxy-tags` script AFTER you resolve the conflicts. + +- `cd` into the assets location output by the script +- resolve the conflict or conflicts +- add the resolution, and invoke `git cherry-pick --continue` + +Afterwards, re-invoke the `merge-proxy-tags` script with arguments given to you in original error. This will leave the assets in a `touched` state that can be `test-proxy push`-ed. diff --git a/eng/common/testproxy/scripts/resolve-asset-conflict/resolve-asset-conflict.ps1 b/eng/common/testproxy/scripts/resolve-asset-conflict/resolve-asset-conflict.ps1 new file mode 100644 index 000000000000..c11e9c879e6c --- /dev/null +++ b/eng/common/testproxy/scripts/resolve-asset-conflict/resolve-asset-conflict.ps1 @@ -0,0 +1,79 @@ +#Requires -Version 6.0 +#Requires -PSEdition Core + +<# +.SYNOPSIS +Within an assets.json file that has in a conflicted state (specifically on asset tag), merge the two tags that are conflicting and leave the assets.json in a commitable state. + +.DESCRIPTION +USAGE: resolve-asset-conflict.ps1 path/to/target_assets_json + +Parses the assets.json file and determines which tags are in conflict. If there are no conflicts, the script exits. + +1. Parse the tags (base and target) from conflicting assets.json. +2. Update the assets.json with the base tag, but remember the target tag. +3. merge-proxy-tags.ps1 $AssetsJson base_tag target_tag + +This script requires that test-proxy or azure.sdk.tools.testproxy should be on the PATH. + +.PARAMETER AssetsJson +The script uses a target assets.json to understand what tags are in conflict. This is the only required parameter. +#> + +param( + [Parameter(Position=0)] + [string] $AssetsJson +) + +. (Join-Path $PSScriptRoot ".." ".." "onboarding" "common-asset-functions.ps1") +. (Join-Path $PSScriptRoot ".." ".." ".." "scripts" "Helpers" "git-helpers.ps1") + +$TestProxy = Resolve-Proxy + +if (!(Test-Path $AssetsJson)) { + Write-Error "AssetsJson file does not exist: $AssetsJson" + exit 1 +} + +# normally we we would Resolve-Path the $AssetsJson, but the git show command only works with relative paths, so we'll just keep that here. +if (-not $AssetsJson.EndsWith("assets.json")) { + Write-Error "This script can only resolve conflicts within an assets.json. The file provided is not an assets.json: $AssetsJson" + exit 1 +} + +$conflictingAssets = [ConflictedFile]::new($AssetsJson) + +if (-not $conflictingAssets.IsConflicted) { + Write-Host "No conflicts found in $AssetsJson, nothing to resolve, so there is no second tag to merge. Exiting" + exit 0 +} + +# this is very dumb, but will properly work! +try { + $BaseAssets = $conflictingAssets.Left() | ConvertFrom-Json +} +catch { + Write-Error "Failed to convert previous version to valid JSON format." + exit 1 +} + +try { + $TargetAssets = $conflictingAssets.Right() | ConvertFrom-Json +} +catch { + Write-Error "Failed to convert target assets.json version to valid JSON format." + exit 1 +} + +Write-Host "Replacing conflicted assets.json with base branch version." -ForegroundColor Green +Set-Content -Path $AssetsJson -Value $conflictingAssets.Left() + +$ScriptPath = Join-Path $PSScriptRoot ".." "tag-merge" "merge-proxy-tags.ps1" +& $ScriptPath $AssetsJson $BaseAssets.Tag $TargetAssets.Tag + +if ($lastexitcode -eq 0) { + Write-Host "Successfully auto-merged assets tag '$($TargetASsets.Tag)' into tag '$($BaseAssets.Tag)'. Invoke 'test-proxy push -a $AssetsJson' and commit the resulting assets.json!" -ForegroundColor Green +} +else { + Write-Host "Conflicts were discovered, resolve the conflicts and invoke the `"merge-proxy-tags.ps1`" as recommended in the line directly above." +} \ No newline at end of file diff --git a/eng/common/testproxy/scripts/tag-merge/README.md b/eng/common/testproxy/scripts/tag-merge/README.md index d72dd4dbefac..d16dc3be054e 100644 --- a/eng/common/testproxy/scripts/tag-merge/README.md +++ b/eng/common/testproxy/scripts/tag-merge/README.md @@ -16,7 +16,7 @@ This script merely allows the abstraction of some of this "combination" work. - Must have the `test-proxy` available on your PATH - `test-proxy` is honored when the proxy is installed as a `dotnet tool` - `Azure.Sdk.Tools.TestProxy` is honored when the standalone executable is on your PATH - - Preference for `dotnet tool` if present + - Defaults to `dotnet tool` if both are present on the PATH. ### Call the script diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index 18e57acc589f..715e4b177538 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -68,24 +68,6 @@ function Git-Command($CommandString, $WorkingDirectory, $HardExit=$true) { return $result.Output } -function Resolve-Proxy { - $testProxyExe = "test-proxy" - # this script requires the presence of the test-proxy on the PATH - $proxyToolPresent = Test-Exe-In-Path -ExeToLookFor "test-proxy" -ExitOnError $false - $proxyStandalonePresent = Test-Exe-In-Path -ExeToLookFor "Azure.Sdk.Tools.TestProxy" -ExitOnError $false - - if (-not $proxyToolPresent -and -not $proxyStandalonePresent) { - Write-Error "This script requires the presence of a test-proxy executable to complete its operations. Exiting." - exit 1 - } - - if (-not $proxyToolPresent) { - $testProxyExe = "Azure.Sdk.Tools.TestProxy" - } - - return $testProxyExe -} - function Call-Proxy { param( [string] $TestProxyExe, @@ -256,14 +238,17 @@ function Prepare-Assets($ProxyExe, $MountDirectory, $AssetsJson) { } } -function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ +function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory, $RelativeAssetsJson){ + $remainingTagString = $RemainingTags -join " " foreach($Tag in $RemainingTags) { $tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation $existingTags = Save-Incomplete-Progress $Tag $MountDirectory $cherryPickResult = Git-Command-With-Result "cherry-pick $tagSha" - $AssetsRepoLocation -HardExit $false if ($cherryPickResult.ExitCode -ne 0) { - Write-Host "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", and re-run this script with the same arguments as before." -ForegroundColor Red + $error = "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", and re-invoke " + + "by `"./eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 $RelativeAssetsJson $remainingTagString`"" + Write-Host $error -ForegroundColor Red exit 1 } } @@ -294,6 +279,7 @@ if ($PSVersionTable["PSVersion"].Major -lt 6) { # resolve the proxy location so that we can invoke it easily, if not present we exit here. $proxyExe = Resolve-Proxy +$relativeAssetsJson = $AssetsJson $AssetsJson = Resolve-Path $AssetsJson # figure out where the root of the repo for the passed assets.json is. We need it to properly set the mounting @@ -313,6 +299,6 @@ $tags = Resolve-Target-Tags $AssetsJson $TargetTags $mountDirectory Start-Message $AssetsJson $Tags $AssetsRepoLocation $mountDirectory -$CombinedTags = Combine-Tags $Tags $AssetsRepoLocation $mountDirectory +$CombinedTags = Combine-Tags $Tags $AssetsRepoLocation $mountDirectory $relativeAssetsJson Finish-Message $AssetsJson $CombinedTags $AssetsRepoLocation $mountDirectory