From d0dc9e9a3622f649335474176cce7e7e40e5a138 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Thu, 30 Mar 2023 23:55:54 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 5608 (#20348) * ongoing * Move get-codeowners scripts and tests to their own dirs * address PR remarks * Fix CodeOwnerFileLocation path, fix casing, and dedup param defaults * fix param names * add todos on needed changes in cpp repo * Add CodeownersFileLocation to Get-Codeowners and use $null for default param values * move get-codeowners back to scripts/ and rename -functions to .lib * fix: use empty string as defaults instead of $nulls, to fix invocation * fix bug with invocation of Get-Codeowners + add support for passing IncludeNonUserAliases as switch * fix path iin Metadata-Helpers.ps1 * fix typo --------- Co-authored-by: Konrad Jamrozik --- .../stages/archetype-sdk-tool-pwsh.yml | 13 +- .../templates/steps/run-pester-tests.yml | 20 +- eng/common/scripts/get-codeowners.lib.ps1 | 133 +++++++++++++ eng/common/scripts/get-codeowners.ps1 | 181 ++---------------- 4 files changed, 162 insertions(+), 185 deletions(-) create mode 100644 eng/common/scripts/get-codeowners.lib.ps1 diff --git a/eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml b/eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml index e01694adf034..5d78b132e26d 100644 --- a/eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml +++ b/eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml @@ -15,20 +15,21 @@ variables: - template: /eng/pipelines/templates/variables/globals.yml stages: - - stage: 'eng_script_tests' + - stage: + displayName: Run PowerShell Tests jobs: - - job: 'Test' + - job: Test strategy: matrix: Windows: - Pool: 'azsdk-pool-mms-win-2022-general' - Image: 'windows-2022' + Pool: azsdk-pool-mms-win-2022-general + Image: windows-2022 Linux: Pool: azsdk-pool-mms-ubuntu-2204-general Image: MMSUbuntu22.04 Mac: - Pool: 'Azure Pipelines' - Image: 'macos-11' + Pool: Azure Pipelines + Image: macos-11 pool: name: $(Pool) diff --git a/eng/common/pipelines/templates/steps/run-pester-tests.yml b/eng/common/pipelines/templates/steps/run-pester-tests.yml index 02082a6470b7..ce42f65b6466 100644 --- a/eng/common/pipelines/templates/steps/run-pester-tests.yml +++ b/eng/common/pipelines/templates/steps/run-pester-tests.yml @@ -13,7 +13,7 @@ parameters: steps: - pwsh: | - Install-Module -Name Pester -Force + Install-Module -Name Pester -Force displayName: Install Pester # default test steps @@ -30,8 +30,10 @@ steps: $config.Filter.Tag = $tags } + Write-Host "Calling 'Invoke-Pester' in workingDirectory '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}'. Pester tags filter: '$tags'" + # https://pester.dev/docs/commands/Invoke-Pester Invoke-Pester -Configuration $config - displayName: Run Tests + displayName: Run Pester Tests env: ${{ parameters.EnvVars }} workingDirectory: $(Build.SourcesDirectory)/${{ parameters.TargetDirectory }} @@ -39,17 +41,17 @@ steps: - ${{ parameters.CustomTestSteps }} - task: PublishTestResults@2 - displayName: 'Publish Test Results' + displayName: Publish Test Results condition: succeededOrFailed() inputs: - testResultsFormat: 'NUnit' + testResultsFormat: NUnit testResultsFiles: $(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}/testResults.xml - testRunTitle: '$(System.StageName)_$(Agent.JobName)_Tests' + testRunTitle: $(System.StageName)_$(Agent.JobName)_Tests - task: PublishCodeCoverageResults@1 - displayName: 'Publish Code Coverage to Azure DevOps' + displayName: Publish Code Coverage to Azure DevOps condition: succeededOrFailed() inputs: - codeCoverageTool: 'JaCoCo' - summaryFileLocation: '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}/coverage.xml' - pathToSources: '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}' \ No newline at end of file + codeCoverageTool: JaCoCo + summaryFileLocation: $(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}/coverage.xml + pathToSources: $(Build.SourcesDirectory)/${{ parameters.TargetDirectory }} \ No newline at end of file diff --git a/eng/common/scripts/get-codeowners.lib.ps1 b/eng/common/scripts/get-codeowners.lib.ps1 new file mode 100644 index 000000000000..c34f28bb225b --- /dev/null +++ b/eng/common/scripts/get-codeowners.lib.ps1 @@ -0,0 +1,133 @@ +function Get-CodeownersTool([string] $ToolPath, [string] $DevOpsFeed, [string] $ToolVersion) +{ + $codeownersToolCommand = Join-Path $ToolPath "retrieve-codeowners" + # Check if the retrieve-codeowners tool exists or not. + if (Get-Command $codeownersToolCommand -errorAction SilentlyContinue) { + return $codeownersToolCommand + } + if (!(Test-Path $ToolPath)) { + New-Item -ItemType Directory -Path $ToolPath | Out-Null + } + Write-Host "Installing the retrieve-codeowners tool under tool path: $ToolPath ..." + + # Run command under tool path to avoid dotnet tool install command checking .csproj files. + # This is a bug for dotnet tool command. Issue: https://github.com/dotnet/sdk/issues/9623 + Push-Location $ToolPath + dotnet tool install --tool-path $ToolPath --add-source $DevOpsFeed --version $ToolVersion "Azure.Sdk.Tools.RetrieveCodeOwners" | Out-Null + Pop-Location + # Test to see if the tool properly installed. + if (!(Get-Command $codeownersToolCommand -errorAction SilentlyContinue)) { + Write-Error "The retrieve-codeowners tool is not properly installed. Please check your tool path: $ToolPath" + return + } + return $codeownersToolCommand +} + +<# +.SYNOPSIS +A function that given as input $TargetPath param, returns the owners +of that path, as determined by CODEOWNERS file passed in $CodeownersFileLocation +param. + +.PARAMETER TargetPath +Required*. Path to file or directory whose owners are to be determined from a +CODEOWNERS file. e.g. sdk/core/azure-amqp/ or sdk/core/foo.txt. + +*for backward compatibility, you might provide $TargetDirectory instead. + +.PARAMETER TargetDirectory +Obsolete. Replaced by $TargetPath. Kept for backward-compatibility. +If both $TargetPath and $TargetDirectory are provided, $TargetDirectory is +ignored. + +.PARAMETER CodeownersFileLocation +Optional. An absolute path to the CODEOWNERS file against which the $TargetPath param +will be checked to determine its owners. + +.PARAMETER ToolVersion +Optional. The NuGet package version of the package containing the "retrieve-codeowners" +tool, around which this script is a wrapper. + +.PARAMETER ToolPath +Optional. The place to check the "retrieve-codeowners" tool existence. + +.PARAMETER DevOpsFeed +Optional. The NuGet package feed from which the "retrieve-codeowners" tool is to be installed. + +NuGet feed: +https://dev.azure.com/azure-sdk/public/_artifacts/feed/azure-sdk-for-net/NuGet/Azure.Sdk.Tools.RetrieveCodeOwners + +Pipeline publishing the NuGet package to the feed, "tools - code-owners-parser": +https://dev.azure.com/azure-sdk/internal/_build?definitionId=3188 + +.PARAMETER VsoVariable +Optional. If provided, the determined owners, based on $TargetPath matched against CODEOWNERS file at $CodeownersFileLocation, +will be output to Azure DevOps pipeline log as variable named $VsoVariable. + +Reference: +https://learn.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch +https://learn.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#logging-command-format + +.PARAMETER IncludeNonUserAliases +Optional. Whether to include in the returned owners list aliases that are team aliases, e.g. Azure/azure-sdk-team + +.PARAMETER Test +Optional. Whether to run the script against hard-coded tests. + +#> +function Get-Codeowners( + [string] $TargetPath, + [string] $TargetDirectory, + [string] $ToolPath = (Join-Path ([System.IO.Path]::GetTempPath()) "codeowners-tool"), + [string] $DevOpsFeed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json", + [string] $ToolVersion = "1.0.0-dev.20230306.3", + [string] $VsoVariable = "", + [string] $CodeownersFileLocation = "", + [switch] $IncludeNonUserAliases + ) +{ + if ([string]::IsNullOrWhiteSpace($CodeownersFileLocation)) { + # The $PSScriptRoot is assumed to be azure-sdk-tools/eng/common/scripts/get-codeowners.ps1 + $CodeownersFileLocation = (Resolve-Path $PSScriptRoot/../../../.github/CODEOWNERS) + } + + # Backward compatibility: if $TargetPath is not provided, fall-back to the legacy $TargetDirectory + if ([string]::IsNullOrWhiteSpace($TargetPath)) { + $TargetPath = $TargetDirectory + } + if ([string]::IsNullOrWhiteSpace($TargetPath)) { + Write-Error "TargetPath (or TargetDirectory) parameter must be neither null nor whitespace." + return ,@() + } + + $codeownersToolCommand = Get-CodeownersTool -ToolPath $ToolPath -DevOpsFeed $DevOpsFeed -ToolVersion $ToolVersion + Write-Host "Executing: & $codeownersToolCommand --target-path $TargetPath --codeowners-file-path-or-url $CodeownersFileLocation --exclude-non-user-aliases:$(!$IncludeNonUserAliases)" + $commandOutput = & $codeownersToolCommand ` + --target-path $TargetPath ` + --codeowners-file-path-or-url $CodeownersFileLocation ` + --exclude-non-user-aliases:$(!$IncludeNonUserAliases) ` + 2>&1 + + if ($LASTEXITCODE -ne 0) { + Write-Host "Command $codeownersToolCommand execution failed (exit code = $LASTEXITCODE). Output string: $commandOutput" + return ,@() + } else + { + Write-Host "Command $codeownersToolCommand executed successfully (exit code = 0). Command output string length: $($commandOutput.length)" + } + +# Assert: $commandOutput is a valid JSON representing: +# - a single CodeownersEntry, if the $TargetPath was a single path +# - or a dictionary of CodeownerEntries, keyes by each path resolved from a $TargetPath glob path. +# +# For implementation details, see Azure.Sdk.Tools.RetrieveCodeOwners.Program.Main + +$codeownersJson = $commandOutput | ConvertFrom-Json + + if ($VsoVariable) { + $codeowners = $codeownersJson.Owners -join "," + Write-Host "##vso[task.setvariable variable=$VsoVariable;]$codeowners" + } + + return ,@($codeownersJson.Owners) +} \ No newline at end of file diff --git a/eng/common/scripts/get-codeowners.ps1 b/eng/common/scripts/get-codeowners.ps1 index a4a18ecea42f..b40f9b4fa29f 100644 --- a/eng/common/scripts/get-codeowners.ps1 +++ b/eng/common/scripts/get-codeowners.ps1 @@ -1,177 +1,18 @@ <# .SYNOPSIS -A script that given as input $TargetPath param, returns the owners -of that path, as determined by CODEOWNERS file passed in $CodeOwnersFileLocation -param. - -.PARAMETER TargetPath -Required*. Path to file or directory whose owners are to be determined from a -CODEOWNERS file. e.g. sdk/core/azure-amqp/ or sdk/core/foo.txt. - -*for backward compatibility, you might provide $TargetDirectory instead. - -.PARAMETER TargetDirectory -Obsolete. Replaced by $TargetPath. Kept for backward-compatibility. -If both $TargetPath and $TargetDirectory are provided, $TargetDirectory is -ignored. - -.PARAMETER CodeOwnerFileLocation -Optional. An absolute path to the CODEOWNERS file against which the $TargetPath param -will be checked to determine its owners. - -.PARAMETER ToolVersion -Optional. The NuGet package version of the package containing the "retrieve-codeowners" -tool, around which this script is a wrapper. - -.PARAMETER ToolPath -Optional. The place to check the "retrieve-codeowners" tool existence. - -.PARAMETER DevOpsFeed -Optional. The NuGet package feed from which the "retrieve-codeowners" tool is to be installed. - -NuGet feed: -https://dev.azure.com/azure-sdk/public/_artifacts/feed/azure-sdk-for-net/NuGet/Azure.Sdk.Tools.RetrieveCodeOwners - -Pipeline publishing the NuGet package to the feed, "tools - code-owners-parser": -https://dev.azure.com/azure-sdk/internal/_build?definitionId=3188 - -.PARAMETER VsoVariable -Optional. If provided, the determined owners, based on $TargetPath matched against CODEOWNERS file at $CodeOwnerFileLocation, -will be output to Azure DevOps pipeline log as variable named $VsoVariable. - -Reference: -https://learn.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch -https://learn.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#logging-command-format - -.PARAMETER IncludeNonUserAliases -Optional. Whether to include in the returned owners list aliases that are team aliases, e.g. Azure/azure-sdk-team - -.PARAMETER Test -Optional. Whether to run the script against hard-coded tests. - +Please see the comment on Get-Codeowners defined in ./get-codeowners.lib.ps1 #> param ( - [string]$TargetPath = "", - [string]$TargetDirectory = "", - # The path used assumes the script is located in azure-sdk-tools/eng/common/scripts/get-codeowners.ps1 - [string]$CodeOwnerFileLocation = (Resolve-Path $PSScriptRoot/../../../.github/CODEOWNERS), - # The $ToolVersion 1.0.0-dev.20230214.3 includes following PR: - # Use CodeownersFile.UseRegexMatcherDefault everywhere where applicable + remove obsolete tests - # https://github.com/Azure/azure-sdk-tools/pull/5437 - # - # but not this one: - # Remove the obsolete, prefix-based CODEOWNERS matcher & related tests - # https://github.com/Azure/azure-sdk-tools/pull/5431 - [string]$ToolVersion = "1.0.0-dev.20230223.4", - [string]$ToolPath = (Join-Path ([System.IO.Path]::GetTempPath()) "codeowners-tool-path"), - [string]$DevOpsFeed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json", - [string]$VsoVariable = "", - [switch]$IncludeNonUserAliases, - [switch]$Test + [string] $TargetPath = "", + [string] $TargetDirectory = "", + [string] $CodeownersFileLocation = "", + [switch] $IncludeNonUserAliases ) -function Get-CodeownersTool() -{ - $codeownersToolCommand = Join-Path $ToolPath "retrieve-codeowners" - # Check if the retrieve-codeowners tool exists or not. - if (Get-Command $codeownersToolCommand -errorAction SilentlyContinue) { - return $codeownersToolCommand - } - if (!(Test-Path $ToolPath)) { - New-Item -ItemType Directory -Path $ToolPath | Out-Null - } - Write-Host "Installing the retrieve-codeowners tool under tool path: $ToolPath ..." - - # Run command under tool path to avoid dotnet tool install command checking .csproj files. - # This is a bug for dotnet tool command. Issue: https://github.com/dotnet/sdk/issues/9623 - Push-Location $ToolPath - dotnet tool install --tool-path $ToolPath --add-source $DevOpsFeed --version $ToolVersion "Azure.Sdk.Tools.RetrieveCodeOwners" | Out-Null - Pop-Location - # Test to see if the tool properly installed. - if (!(Get-Command $codeownersToolCommand -errorAction SilentlyContinue)) { - Write-Error "The retrieve-codeowners tool is not properly installed. Please check your tool path: $ToolPath" - return - } - return $codeownersToolCommand -} - -function Get-Codeowners( - [string]$targetPath, - [string]$targetDirectory, - [string]$codeownersFileLocation, - [bool]$includeNonUserAliases = $false) -{ - # Backward compaitiblity: if $targetPath is not provided, fall-back to the legacy $targetDirectory - if ([string]::IsNullOrWhiteSpace($targetPath)) { - $targetPath = $targetDirectory - } - if ([string]::IsNullOrWhiteSpace($targetPath)) { - Write-Error "TargetPath (or TargetDirectory) parameter must be neither null nor whitespace." - return ,@() - } - - $codeownersToolCommand = Get-CodeownersTool - Write-Host "Executing: & $codeownersToolCommand --target-path $targetPath --codeowners-file-path-or-url $codeownersFileLocation --exclude-non-user-aliases:$(!$includeNonUserAliases)" - $commandOutput = & $codeownersToolCommand ` - --target-path $targetPath ` - --codeowners-file-path-or-url $codeownersFileLocation ` - --exclude-non-user-aliases:$(!$includeNonUserAliases) ` - 2>&1 - - if ($LASTEXITCODE -ne 0) { - Write-Host "Command $codeownersToolCommand execution failed (exit code = $LASTEXITCODE). Output string: $commandOutput" - return ,@() - } else - { - Write-Host "Command $codeownersToolCommand executed successfully (exit code = 0). Output string length: $($commandOutput.length)" - } - -# Assert: $commandOutput is a valid JSON representing: -# - a single CodeownersEntry, if the $targetPath was a single path -# - or a dictionary of CodeownerEntries, keyes by each path resolved from a $targetPath glob path. -# -# For implementation details, see Azure.Sdk.Tools.RetrieveCodeOwners.Program.Main - -$codeownersJson = $commandOutput | ConvertFrom-Json - - if ($VsoVariable) { - $codeowners = $codeownersJson.Owners -join "," - Write-Host "##vso[task.setvariable variable=$VsoVariable;]$codeowners" - } - - return ,@($codeownersJson.Owners) -} - -function TestGetCodeowners([string]$targetPath, [string]$codeownersFileLocation, [bool]$includeNonUserAliases = $false, [string[]]$expectReturn) { - Write-Host "Test: find owners matching '$targetPath' ..." - - $actualReturn = Get-Codeowners -targetPath $targetPath -codeownersFileLocation $codeownersFileLocation -includeNonUserAliases $IncludeNonUserAliases - - if ($actualReturn.Count -ne $expectReturn.Count) { - Write-Error "The length of actual result is not as expected. Expected length: $($expectReturn.Count), Actual length: $($actualReturn.Count)." - exit 1 - } - for ($i = 0; $i -lt $expectReturn.Count; $i++) { - if ($expectReturn[$i] -ne $actualReturn[$i]) { - Write-Error "Expect result $expectReturn[$i] is different than actual result $actualReturn[$i]." - exit 1 - } - } -} - -if ($Test) { - # Most of tests here have been removed; now instead we should run tests from RetrieveCodeOwnersProgramTests, and in a way as explained in: - # https://github.com/Azure/azure-sdk-tools/issues/5434 - # https://github.com/Azure/azure-sdk-tools/pull/5103#discussion_r1068680818 - Write-Host "Running reduced test suite at `$PSScriptRoot of $PSSCriptRoot. Please see https://github.com/Azure/azure-sdk-tools/issues/5434 for more." +. $PSScriptRoot/get-codeowners.lib.ps1 - $azSdkToolsCodeowners = (Resolve-Path "$PSScriptRoot/../../../.github/CODEOWNERS") - TestGetCodeowners -targetPath "eng/common/scripts/get-codeowners.ps1" -codeownersFileLocation $azSdkToolsCodeowners -includeNonUserAliases $true -expectReturn @("konrad-jamrozik", "weshaggard", "benbp") - - $testCodeowners = (Resolve-Path "$PSScriptRoot/../../../tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/test_CODEOWNERS") - TestGetCodeowners -targetPath "tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/a.txt" -codeownersFileLocation $testCodeowners -includeNonUserAliases $true -expectReturn @("2star") - exit 0 -} -else { - return Get-Codeowners -targetPath $TargetPath -targetDirectory $TargetDirectory -codeownersFileLocation $CodeOwnerFileLocation -includeNonUserAliases $IncludeNonUserAliases -} +return Get-Codeowners ` + -TargetPath $TargetPath ` + -TargetDirectory $TargetDirectory ` + -CodeownersFileLocation $CodeownersFileLocation ` + -IncludeNonUserAliases:$IncludeNonUserAliases \ No newline at end of file