From 14052b10101e61a1561349587797be77e16395df Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 14:44:59 +0100 Subject: [PATCH 1/9] [vs16.11] Update dependencies from dotnet/arcade (#7332) * Update dependencies from https://github.com/dotnet/arcade build 20220105.3 Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.21505.11 -> To Version 5.0.0-beta.22055.3 * Update dependencies from https://github.com/dotnet/arcade build 20220204.7 Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.21505.11 -> To Version 5.0.0-beta.22104.7 * Update dependencies from https://github.com/dotnet/arcade build 20220222.5 Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.21505.11 -> To Version 5.0.0-beta.22122.5 * Update dependencies from https://github.com/dotnet/arcade build 20220223.4 Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.21505.11 -> To Version 5.0.0-beta.22123.4 * update mac version Co-authored-by: dotnet-maestro[bot] Co-authored-by: Forgind --- .vsts-dotnet-ci.yml | 2 +- eng/Version.Details.xml | 4 +- eng/common/pipeline-logging-functions.ps1 | 104 ++++++++++------- eng/common/sdl/configure-sdl-tool.ps1 | 109 ++++++++++++++++++ eng/common/sdl/execute-all-sdl-tools.ps1 | 80 +++++++++---- eng/common/sdl/extract-artifact-archives.ps1 | 63 ++++++++++ eng/common/sdl/init-sdl.ps1 | 1 - eng/common/sdl/packages.config | 2 +- eng/common/sdl/run-sdl.ps1 | 50 +++----- eng/common/templates/job/execute-sdl.yml | 4 +- eng/common/templates/job/job.yml | 5 + eng/common/templates/job/onelocbuild.yml | 2 +- .../templates/job/publish-build-assets.yml | 4 - eng/common/templates/jobs/jobs.yml | 6 +- .../templates/post-build/common-variables.yml | 4 - eng/common/tools.ps1 | 51 ++++++++ global.json | 2 +- 17 files changed, 370 insertions(+), 123 deletions(-) create mode 100644 eng/common/sdl/configure-sdl-tool.ps1 create mode 100644 eng/common/sdl/extract-artifact-archives.ps1 diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 8a7978a2291..d4f3f665982 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -187,7 +187,7 @@ jobs: - job: CoreOnMac displayName: "macOS Core" pool: - vmImage: 'macOS-10.14' + vmImage: 'macOS-latest' steps: - bash: . 'eng/cibuild_bootstrapped_msbuild.sh' displayName: CI Build diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ebeca639894..68fa0c98d10 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 9661b5187c30b8bbd683aea20694909eebcdcb73 + 295d305a5520815cbf4ccb3f209f6ee8ba11b45d https://github.com/nuget/nuget.client diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index 8484451f3a5..8e422c561e4 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -29,14 +29,14 @@ function Write-PipelineTelemetryError { [switch]$AsOutput, [switch]$Force) - $PSBoundParameters.Remove('Category') | Out-Null + $PSBoundParameters.Remove('Category') | Out-Null - if($Force -Or ((Test-Path variable:ci) -And $ci)) { - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" - } - $PSBoundParameters.Remove('Message') | Out-Null - $PSBoundParameters.Add('Message', $Message) - Write-PipelineTaskError @PSBoundParameters + if ($Force -Or ((Test-Path variable:ci) -And $ci)) { + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + } + $PSBoundParameters.Remove('Message') | Out-Null + $PSBoundParameters.Add('Message', $Message) + Write-PipelineTaskError @PSBoundParameters } # Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set @@ -55,8 +55,8 @@ function Write-PipelineTaskError { [switch]$Force ) - if(!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { - if($Type -eq 'error') { + if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { + if ($Type -eq 'error') { Write-Host $Message -ForegroundColor Red return } @@ -66,47 +66,61 @@ function Write-PipelineTaskError { } } - if(($Type -ne 'error') -and ($Type -ne 'warning')) { + if (($Type -ne 'error') -and ($Type -ne 'warning')) { Write-Host $Message return } $PSBoundParameters.Remove('Force') | Out-Null - if(-not $PSBoundParameters.ContainsKey('Type')) { + if (-not $PSBoundParameters.ContainsKey('Type')) { $PSBoundParameters.Add('Type', 'error') } Write-LogIssue @PSBoundParameters - } +} - function Write-PipelineSetVariable { +function Write-PipelineSetVariable { [CmdletBinding()] param( - [Parameter(Mandatory = $true)] - [string]$Name, - [string]$Value, - [switch]$Secret, - [switch]$AsOutput, - [bool]$IsMultiJobVariable=$true) - - if((Test-Path variable:ci) -And $ci) { + [Parameter(Mandatory = $true)] + [string]$Name, + [string]$Value, + [switch]$Secret, + [switch]$AsOutput, + [bool]$IsMultiJobVariable = $true) + + if ((Test-Path variable:ci) -And $ci) { Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ - 'variable' = $Name - 'isSecret' = $Secret - 'isOutput' = $IsMultiJobVariable + 'variable' = $Name + 'isSecret' = $Secret + 'isOutput' = $IsMultiJobVariable } -AsOutput:$AsOutput - } - } + } +} - function Write-PipelinePrependPath { +function Write-PipelinePrependPath { [CmdletBinding()] param( - [Parameter(Mandatory=$true)] - [string]$Path, - [switch]$AsOutput) + [Parameter(Mandatory = $true)] + [string]$Path, + [switch]$AsOutput) - if((Test-Path variable:ci) -And $ci) { + if ((Test-Path variable:ci) -And $ci) { Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput - } - } + } +} + +function Write-PipelineSetResult { + [CmdletBinding()] + param( + [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")] + [Parameter(Mandatory = $true)] + [string]$Result, + [string]$Message) + if ((Test-Path variable:ci) -And $ci) { + Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{ + 'result' = $Result + } + } +} <######################################## # Private functions. @@ -123,7 +137,8 @@ function Format-LoggingCommandData { foreach ($mapping in $script:loggingCommandEscapeMappings) { $Value = $Value.Replace($mapping.Token, $mapping.Replacement) } - } else { + } + else { for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { $mapping = $script:loggingCommandEscapeMappings[$i] $Value = $Value.Replace($mapping.Replacement, $mapping.Token) @@ -156,7 +171,8 @@ function Format-LoggingCommand { if ($first) { $null = $sb.Append(' ') $first = $false - } else { + } + else { $null = $sb.Append(';') } @@ -193,7 +209,8 @@ function Write-LoggingCommand { $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties if ($AsOutput) { $command - } else { + } + else { Write-Host $command } } @@ -212,12 +229,12 @@ function Write-LogIssue { [switch]$AsOutput) $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ - 'type' = $Type - 'code' = $ErrCode - 'sourcepath' = $SourcePath - 'linenumber' = $LineNumber - 'columnnumber' = $ColumnNumber - } + 'type' = $Type + 'code' = $ErrCode + 'sourcepath' = $SourcePath + 'linenumber' = $LineNumber + 'columnnumber' = $ColumnNumber + } if ($AsOutput) { return $command } @@ -229,7 +246,8 @@ function Write-LogIssue { $foregroundColor = [System.ConsoleColor]::Red $backgroundColor = [System.ConsoleColor]::Black } - } else { + } + else { $foregroundColor = $host.PrivateData.WarningForegroundColor $backgroundColor = $host.PrivateData.WarningBackgroundColor if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1 new file mode 100644 index 00000000000..8a68fc24b11 --- /dev/null +++ b/eng/common/sdl/configure-sdl-tool.ps1 @@ -0,0 +1,109 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + # The list of Guardian tools to configure. For each object in the array: + # - If the item is a [hashtable], it must contain these entries: + # - Name = The tool name as Guardian knows it. + # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique + # among all tool entries with the same Name. + # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")' + # - If the item is a [string] $v, it is treated as '@{ Name="$v" }' + [object[]] $ToolsList, + [string] $GuardianLoggerLevel='Standard', + # Optional: Additional params to add to any tool using CredScan. + [string[]] $CrScanAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using PoliCheck. + [string[]] $PoliCheckAdditionalRunConfigParams +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + # Normalize tools list: all in [hashtable] form with defined values for each key. + $ToolsList = $ToolsList | + ForEach-Object { + if ($_ -is [string]) { + $_ = @{ Name = $_ } + } + + if (-not ($_['Scenario'])) { $_.Scenario = "" } + if (-not ($_['Args'])) { $_.Args = @() } + $_ + } + + Write-Host "List of tools to configure:" + $ToolsList | ForEach-Object { $_ | Out-String | Write-Host } + + # We store config files in the r directory of .gdn + $gdnConfigPath = Join-Path $GdnFolder 'r' + $ValidPath = Test-Path $GuardianCliLocation + + if ($ValidPath -eq $False) + { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." + ExitWithExitCode 1 + } + + foreach ($tool in $ToolsList) { + # Put together the name and scenario to make a unique key. + $toolConfigName = $tool.Name + if ($tool.Scenario) { + $toolConfigName += "_" + $tool.Scenario + } + + Write-Host "=== Configuring $toolConfigName..." + + $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" + + # For some tools, add default and automatic args. + if ($tool.Name -eq 'credscan') { + if ($targetDirectory) { + $tool.Args += "`"TargetDirectory < $TargetDirectory`"" + } + $tool.Args += "`"OutputType < pre`"" + $tool.Args += $CrScanAdditionalRunConfigParams + } elseif ($tool.Name -eq 'policheck') { + if ($targetDirectory) { + $tool.Args += "`"Target < $TargetDirectory`"" + } + $tool.Args += $PoliCheckAdditionalRunConfigParams + } + + # Create variable pointing to the args array directly so we can use splat syntax later. + $toolArgs = $tool.Args + + # Configure the tool. If args array is provided or the current tool has some default arguments + # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}", + # one per parameter. Doc page for "guardian configure": + # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure + Exec-BlockVerbosely { + & $GuardianCliLocation configure ` + --working-directory $WorkingDirectory ` + --tool $tool.Name ` + --output-path $gdnConfigFile ` + --logger-level $GuardianLoggerLevel ` + --noninteractive ` + --force ` + $(if ($toolArgs) { "--args" }) @toolArgs + Exit-IfNZEC "Sdl" + } + + Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile" + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index b6a7de26b47..e5bef8ebd3a 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -7,8 +7,17 @@ Param( [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault - [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code - [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + + # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list + # format. + [object[]] $SourceToolsList, + # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools + # list format. + [object[]] $ArtifactToolsList, + # Optional: list of SDL tools to run without automatically specifying a target directory. See + # 'configure-sdl-tool.ps1' for tools list format. + [object[]] $CustomToolsList, + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. @@ -63,13 +72,16 @@ try { ExitWithExitCode 1 } - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + } $gdnFolder = Join-Path $workingDirectory '.gdn' if ($TsaOnboard) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + Exec-BlockVerbosely { + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } if ($LASTEXITCODE -ne 0) { Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE." ExitWithExitCode $LASTEXITCODE @@ -80,24 +92,41 @@ try { } } - if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams - if ($LASTEXITCODE -ne 0) { - ExitWithExitCode $LASTEXITCODE - } - } - if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams - if ($LASTEXITCODE -ne 0) { - ExitWithExitCode $LASTEXITCODE + # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory. + function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) { + if ($tools -and $tools.Count -gt 0) { + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') ` + -GuardianCliLocation $guardianCliLocation ` + -WorkingDirectory $workingDirectory ` + -TargetDirectory $targetDirectory ` + -GdnFolder $gdnFolder ` + -ToolsList $tools ` + -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` + -GuardianLoggerLevel $GuardianLoggerLevel ` + -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` + -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + if ($BreakOnFailure) { + Exit-IfNZEC "Sdl" + } + } } } - if ($UpdateBaseline) { - & (Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Update baseline' - if ($LASTEXITCODE -ne 0) { - ExitWithExitCode $LASTEXITCODE - } + # Configure Artifact and Source tools with default Target directories. + Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory + Configure-ToolsList $SourceToolsList $SourceDirectory + # Configure custom tools with no default Target directory. + Configure-ToolsList $CustomToolsList $null + + # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call. + # (If we used "run" multiple times, each run would overwrite data from earlier runs.) + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'run-sdl.ps1') ` + -GuardianCliLocation $guardianCliLocation ` + -WorkingDirectory $SourceDirectory ` + -UpdateBaseline $UpdateBaseline ` + -GdnFolder $gdnFolder } if ($TsaPublish) { @@ -105,8 +134,9 @@ try { if (-not $TsaRepositoryName) { $TsaRepositoryName = "$($Repository)-$($BranchName)" } - Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + Exec-BlockVerbosely { + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } if ($LASTEXITCODE -ne 0) { Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE." ExitWithExitCode $LASTEXITCODE @@ -119,7 +149,11 @@ try { if ($BreakOnFailure) { Write-Host "Failing the build in case of breaking results..." - & $guardianCliLocation break + Exec-BlockVerbosely { + & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } + } else { + Write-Host "Letting the build pass even if there were breaking results..." } } catch { diff --git a/eng/common/sdl/extract-artifact-archives.ps1 b/eng/common/sdl/extract-artifact-archives.ps1 new file mode 100644 index 00000000000..68da4fbf257 --- /dev/null +++ b/eng/common/sdl/extract-artifact-archives.ps1 @@ -0,0 +1,63 @@ +# This script looks for each archive file in a directory and extracts it into the target directory. +# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**". +# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip. +param( + # Full path to directory where archives are stored. + [Parameter(Mandatory=$true)][string] $InputPath, + # Full path to directory to extract archives into. May be the same as $InputPath. + [Parameter(Mandatory=$true)][string] $ExtractPath +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +$disableConfigureToolsetImport = $true + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + Measure-Command { + $jobs = @() + + # Find archive files for non-Windows and Windows builds. + $archiveFiles = @( + Get-ChildItem (Join-Path $InputPath "*.tar.gz") + Get-ChildItem (Join-Path $InputPath "*.zip") + ) + + foreach ($targzFile in $archiveFiles) { + $jobs += Start-Job -ScriptBlock { + $file = $using:targzFile + $fileName = [System.IO.Path]::GetFileName($file) + $extractDir = Join-Path $using:ExtractPath "$fileName.extracted" + + New-Item $extractDir -ItemType Directory -Force | Out-Null + + Write-Host "Extracting '$file' to '$extractDir'..." + + # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early. + # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the + # error. Save output so it can be stored in the exception string along with context. + $output = tar -xf $file -C $extractDir 2>&1 + # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we + # don't have access to the outer scope. + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'" + } + + Write-Host "Extracted to $extractDir" + } + } + + Receive-Job $jobs -Wait + } +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index ac1bc4b87bb..3ac1d92b370 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -46,7 +46,6 @@ try { Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE." ExitWithExitCode $LASTEXITCODE } - & $(Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Initialize gdn folder' ExitWithExitCode 0 } catch { diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 3bd8b29ebd7..b7bcfe38caf 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index 3d9c87aba6a..2eac8c78f10 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -1,13 +1,9 @@ Param( [string] $GuardianCliLocation, [string] $WorkingDirectory, - [string] $TargetDirectory, [string] $GdnFolder, - [string[]] $ToolsList, [string] $UpdateBaseline, - [string] $GuardianLoggerLevel='Standard', - [string[]] $CrScanAdditionalRunConfigParams, - [string[]] $PoliCheckAdditionalRunConfigParams + [string] $GuardianLoggerLevel='Standard' ) $ErrorActionPreference = 'Stop' @@ -23,7 +19,6 @@ try { . $PSScriptRoot\..\tools.ps1 # We store config files in the r directory of .gdn - Write-Host $ToolsList $gdnConfigPath = Join-Path $GdnFolder 'r' $ValidPath = Test-Path $GuardianCliLocation @@ -33,37 +28,18 @@ try { ExitWithExitCode 1 } - $configParam = @('--config') - - foreach ($tool in $ToolsList) { - $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" - Write-Host $tool - # We have to manually configure tools that run on source to look at the source directory only - if ($tool -eq 'credscan') { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } - if ($tool -eq 'policheck') { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } - - $configParam+=$gdnConfigFile - } - - Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam" - & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE + $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig' + Write-Host "Discovered Guardian config files:" + $gdnConfigFiles | Out-String | Write-Host + + Exec-BlockVerbosely { + & $GuardianCliLocation run ` + --working-directory $WorkingDirectory ` + --baseline mainbaseline ` + --update-baseline $UpdateBaseline ` + --logger-level $GuardianLoggerLevel ` + --config @gdnConfigFiles + Exit-IfNZEC "Sdl" } } catch { diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 53c100222b2..ccd2a340934 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -28,7 +28,7 @@ jobs: - name: AzDOBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: - name: Hosted VS2017 + vmImage: windows-2019 steps: - checkout: self clean: true @@ -83,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.53.3 + -GuardianPackageName Microsoft.Guardian.Cli.0.110.1 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 2dcda1a8a61..b536c95ff44 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -25,6 +25,7 @@ parameters: enablePublishTestResults: false enablePublishUsingPipelines: false useBuildManifest: false + disableComponentGovernance: false mergeTestResults: false testRunTitle: '' name: '' @@ -136,6 +137,10 @@ jobs: richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin continueOnError: true + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: MicroBuildCleanup@1 diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index e8bc77d2ebb..2b55a567f82 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -4,7 +4,7 @@ parameters: # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex GithubPat: $(BotAccount-dotnet-bot-repo-PAT) diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index d0c3cc2b3ba..f349d7ce980 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -37,10 +37,6 @@ jobs: - name: _BuildConfig value: ${{ parameters.configuration }} - group: Publish-Build-Assets - # Skip component governance and codesign validation for SDL. These jobs - # create no content. - - name: skipComponentGovernanceDetection - value: true - name: runCodesignValidationInjection value: false diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 08845950f44..c1c6e0d611c 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -7,7 +7,7 @@ parameters: # Optional: Enable publishing using release pipelines enablePublishUsingPipelines: false - + graphFileGeneration: # Optional: Enable generating the graph files at the end of the build enabled: false @@ -72,7 +72,7 @@ jobs: - ${{ if eq(parameters.runSourceBuild, true) }}: - Source_Build_Complete pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} @@ -85,4 +85,4 @@ jobs: dependsOn: - Asset_Registry_Publish pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index c99fd750376..956ed7eb904 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -91,9 +91,5 @@ variables: - name: InternalInstallersBlobFeedKey value: $(dotnetclimsrc-access-key) - # Skip component governance and codesign validation for SDL. These jobs - # create no content. - - name: skipComponentGovernanceDetection - value: true - name: runCodesignValidationInjection value: false diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index eba7ed49d78..15b2ef4cec8 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -103,6 +103,46 @@ function Exec-Process([string]$command, [string]$commandArgs) { } } +# Take the given block, print it, print what the block probably references from the current set of +# variables using low-effort string matching, then run the block. +# +# This is intended to replace the pattern of manually copy-pasting a command, wrapping it in quotes, +# and printing it using "Write-Host". The copy-paste method is more readable in build logs, but less +# maintainable and less reliable. It is easy to make a mistake and modify the command without +# properly updating the "Write-Host" line, resulting in misleading build logs. The probability of +# this mistake makes the pattern hard to trust when it shows up in build logs. Finding the bug in +# existing source code can also be difficult, because the strings are not aligned to each other and +# the line may be 300+ columns long. +# +# By removing the need to maintain two copies of the command, Exec-BlockVerbosely avoids the issues. +# +# In Bash (or any posix-like shell), "set -x" prints usable verbose output automatically. +# "Set-PSDebug" appears to be similar at first glance, but unfortunately, it isn't very useful: it +# doesn't print any info about the variables being used by the command, which is normally the +# interesting part to diagnose. +function Exec-BlockVerbosely([scriptblock] $block) { + Write-Host "--- Running script block:" + $blockString = $block.ToString().Trim() + Write-Host $blockString + + Write-Host "--- List of variables that might be used:" + # For each variable x in the environment, check the block for a reference to x via simple "$x" or + # "@x" syntax. This doesn't detect other ways to reference variables ("${x}" nor "$variable:x", + # among others). It only catches what this function was originally written for: simple + # command-line commands. + $variableTable = Get-Variable | + Where-Object { + $blockString.Contains("`$$($_.Name)") -or $blockString.Contains("@$($_.Name)") + } | + Format-Table -AutoSize -HideTableHeaders -Wrap | + Out-String + Write-Host $variableTable.Trim() + + Write-Host "--- Executing:" + & $block + Write-Host "--- Done running script block!" +} + # createSdkLocationFile parameter enables a file being generated under the toolset directory # which writes the sdk's location into. This is only necessary for cmd --> powershell invocations # as dot sourcing isn't possible. @@ -614,6 +654,17 @@ function ExitWithExitCode([int] $exitCode) { exit $exitCode } +# Check if $LASTEXITCODE is a nonzero exit code (NZEC). If so, print a Azure Pipeline error for +# diagnostics, then exit the script with the $LASTEXITCODE. +function Exit-IfNZEC([string] $category = "General") { + Write-Host "Exit code $LASTEXITCODE" + if ($LASTEXITCODE -ne 0) { + $message = "Last command failed with exit code $LASTEXITCODE." + Write-PipelineTelemetryError -Force -Category $category -Message $message + ExitWithExitCode $LASTEXITCODE + } +} + function Stop-Processes() { Write-Host 'Killing running build processes...' foreach ($processName in $processesToStopOnExit) { diff --git a/global.json b/global.json index 33ff6dedce8..d8af08589ec 100644 --- a/global.json +++ b/global.json @@ -12,6 +12,6 @@ }, "msbuild-sdks": { "Microsoft.Build.CentralPackageVersions": "2.0.1", - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.21505.11" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.22123.4" } } From cd34b35347a9cf15b8875c199399ea96d2d78cc2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 10 Mar 2022 13:24:35 +0000 Subject: [PATCH 2/9] Update dependencies from https://github.com/dotnet/arcade build 20220309.5 Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.22123.4 -> To Version 5.0.0-beta.22159.5 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 68fa0c98d10..787e935a75a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 295d305a5520815cbf4ccb3f209f6ee8ba11b45d + ee744068a4fcccc5b8b56e0025f9c95aa19ff318 https://github.com/nuget/nuget.client diff --git a/global.json b/global.json index d8af08589ec..ac20b8f9030 100644 --- a/global.json +++ b/global.json @@ -12,6 +12,6 @@ }, "msbuild-sdks": { "Microsoft.Build.CentralPackageVersions": "2.0.1", - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.22123.4" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.22159.5" } } From e7498c064121704581d9e8af9dd00c3c5a2d52ac Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 27 May 2022 12:43:24 +0000 Subject: [PATCH 3/9] Update dependencies from https://github.com/dotnet/arcade build 20220526.2 Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.22123.4 -> To Version 5.0.0-beta.22276.2 --- eng/Version.Details.xml | 4 ++-- eng/common/templates/post-build/post-build.yml | 1 + global.json | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 787e935a75a..31eb326d23f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - ee744068a4fcccc5b8b56e0025f9c95aa19ff318 + 9c6a04ea1e79e9fcd4e60abd5d2c577075787f93 https://github.com/nuget/nuget.client diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 1b0af40d52f..c3ccd64dd37 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -184,6 +184,7 @@ stages: arguments: -task SigningValidation -restore -msbuildEngine vs /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + /p:CheckEolTargetFramework=false ${{ parameters.signingValidationAdditionalParameters }} - template: ../steps/publish-logs.yml diff --git a/global.json b/global.json index ac20b8f9030..cf3f2800c86 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "5.0.401", + "dotnet": "5.0.408", "runtimes": { "dotnet/x64": [ "2.1.7" @@ -12,6 +12,6 @@ }, "msbuild-sdks": { "Microsoft.Build.CentralPackageVersions": "2.0.1", - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.22159.5" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.22276.2" } } From 9d4295a1c62bcd675458b1a5033b1b04061c72f2 Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Wed, 1 Jun 2022 12:04:09 -0500 Subject: [PATCH 4/9] Bump CLI version to match global.json --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index ac4d55b539e..fdaaeb3194d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -28,7 +28,7 @@ - 5.0.401 + 5.0.408 3.9.0-2.20574.26 5.11.0-rc.10 From 0d8881d0ad5851f31d2f55f5e805a0f7db81682d Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Wed, 1 Jun 2022 12:32:22 -0500 Subject: [PATCH 5/9] Bump NuGet version to match --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index fdaaeb3194d..635d5fbe7f5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -30,7 +30,7 @@ Otherwise, this version of dotnet will not be installed and the build will error out. --> 5.0.408 3.9.0-2.20574.26 - 5.11.0-rc.10 + 5.11.1-rc.5 - 16.9.2release + 16.9.3release 15.1.0.0 preview true @@ -23,7 +23,7 @@ - 3.1.100 + 3.1.120 3.3.1-beta3-final 5.9.0-preview.3.7016 diff --git a/global.json b/global.json index f3705522809..2205bf5fcf5 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "3.1.100", + "dotnet": "3.1.120", "runtimes": { "dotnet/x64": [ "2.1.7" diff --git a/src/Shared/AssemblyNameExtension.cs b/src/Shared/AssemblyNameExtension.cs index 1d4f2a4bf71..40684d749f9 100644 --- a/src/Shared/AssemblyNameExtension.cs +++ b/src/Shared/AssemblyNameExtension.cs @@ -147,7 +147,9 @@ private AssemblyNameExtension(SerializationInfo info, StreamingContext context) var hashAlgorithm = (System.Configuration.Assemblies.AssemblyHashAlgorithm) info.GetInt32("hashAlg"); var versionCompatibility = (AssemblyVersionCompatibility) info.GetInt32("verCompat"); var codeBase = info.GetString("codebase"); +#if NETFRAMEWORK var keyPair = (StrongNameKeyPair) info.GetValue("keypair", typeof(StrongNameKeyPair)); +#endif asAssemblyName = new AssemblyName { @@ -159,7 +161,9 @@ private AssemblyNameExtension(SerializationInfo info, StreamingContext context) HashAlgorithm = hashAlgorithm, VersionCompatibility = versionCompatibility, CodeBase = codeBase, +#if NETFRAMEWORK KeyPair = keyPair +#endif }; asAssemblyName.SetPublicKey(publicKey); @@ -984,7 +988,9 @@ public void GetObjectData(SerializationInfo info, StreamingContext context) info.AddValue("hashAlg", asAssemblyName.HashAlgorithm); info.AddValue("verCompat", asAssemblyName.VersionCompatibility); info.AddValue("codebase", asAssemblyName.CodeBase); +#if NETFRAMEWORK info.AddValue("keypair", asAssemblyName.KeyPair); +#endif } info.AddValue("asStr", asString); From 9cb5d94cce6584c4a3b498741049856fe7235f92 Mon Sep 17 00:00:00 2001 From: AR-May <67507805+AR-May@users.noreply.github.com> Date: Mon, 28 Nov 2022 17:20:08 +0100 Subject: [PATCH 7/9] Configure OptProf v2 pipeline 16.11 (#8189) * Update the runsettings configuration creation for opt-prof v2 pipeline. * Add opt-prof v2 pipeline configuration yml. --- .opt-prof.yml | 84 +++++++++++++++++++++++++++++++ .vsts-dotnet.yml | 1 + eng/config/OptProf.runsettings | 92 ++++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 .opt-prof.yml create mode 100644 eng/config/OptProf.runsettings diff --git a/.opt-prof.yml b/.opt-prof.yml new file mode 100644 index 00000000000..35a5a5382d8 --- /dev/null +++ b/.opt-prof.yml @@ -0,0 +1,84 @@ +# Runs OptProf for MSBuild binaries based on an insertion build + +trigger: none # Prevents this pipeline from triggering on check-ins +pr: none # don't run this on PR as well, this should only trigger off on MSBuild runs +schedules: + - cron: "0 9 * * Wed,Sat" + displayName: Weekly OptProf Run + branches: + include: + - main + always: true + +resources: + pipelines: + - pipeline: ComponentBuildUnderTest + source: MSBuild\MSBuild # The name of the pipeline that produces the artifact + trigger: + branches: + - vs* + - pipeline: DartLab + project: DevDiv + source: DartLab + branch: main + - pipeline: DartLab.OptProf + source: DartLab.OptProf + branch: main + repositories: + - repository: DartLabTemplates + type: git + name: DartLab.Templates + ref: refs/heads/main + - repository: DartLabOptProfTemplates + type: git + name: DartLab.OptProf + ref: refs/heads/main + +stages: +- template: \templates\stages\visual-studio\single-runsettings.yml@DartLabOptProfTemplates + parameters: + ##### Required ##### + runSettingsURI: $(Pipeline.Workspace)\ComponentBuildUnderTest\VSSetup\Insertion\OptProf\Training.runsettings + visualStudioBootstrapperURI: $(VisualStudio.InstallationUnderTest.BootstrapperURL) + ##### Optional ##### + name: OptProfProfilingWorkflow + displayName: OptProf Profiling Workflow + optOptimizationInputsDropName: $(OptimizationInputsDropName) + testLabPoolName: VS-Platform # The test lab pool to run your tests in + testMachineImageName: Windows-10-Enterprise-20H2 + visualStudioSigning: Test + variables: + - name: branchName # The branch in the VS repo the bootstrapper was based on + value: $[replace(variables['resources.pipeline.ComponentBuildUnderTest.sourceBranch'],'refs/heads/','')] + - name: OptimizationInputsDropName # The naming convention of the OptimizationInputsDrop + value: OptimizationData/DotNet-msbuild-Trusted/$(branchName)/$(resources.pipeline.ComponentBuildUnderTest.runName)/$(Build.BuildId)/$(System.StageAttempt) + ##### Step Hooks ##### + preTestMachineConfigurationStepList: + - download: ComponentBuildUnderTest + artifact: MicroBuildOutputs + patterns: '**\BootstrapperInfo.json' + displayName: Download Bootstrapper Information + - download: ComponentBuildUnderTest + artifact: VSSetup + patterns: '**\*.runsettings' + displayName: Download OptProf Information + - powershell: | + try { + $bootstrapperInfoJson = Get-Content -Raw -Path '$(Pipeline.Workspace)\ComponentBuildUnderTest\MicroBuildOutputs\BootstrapperInfo.json' | ConvertFrom-Json + $vsBranch = $bootstrapperInfoJson[0].VSBranch + Write-Host "VSBootstrapperBranch: $vsBranch" + Set-AzurePipelinesVariable 'VSBootstrapperBranch' $vsBranch + } + catch { + Write-Host $_ + Write-Error "Failed to set VSBootstrapperBranch pipeline variable" + throw + } + displayName: Set 'VSBootstrapperBranch' variable + - task: PowerShell@2 + displayName: Set 'VisualStudio.InstallationUnderTest.BootstrapperURL' variable + inputs: + filePath: $(DartLab.Path)\Scripts\VisualStudio\Bootstrapper\Get-BootstrapperURL.ps1 + arguments: -BootstrapperInfoJsonURI '$(Pipeline.Workspace)\ComponentBuildUnderTest\MicroBuildOutputs\BootstrapperInfo.json' -VSBranch '$(VSBootstrapperBranch)' -OutVariableName 'VisualStudio.InstallationUnderTest.BootstrapperURL' + preDeployAndRunTestsStepList: + - download: ComponentBuildUnderTest \ No newline at end of file diff --git a/.vsts-dotnet.yml b/.vsts-dotnet.yml index 972a5a00b3e..d76ea236266 100644 --- a/.vsts-dotnet.yml +++ b/.vsts-dotnet.yml @@ -139,6 +139,7 @@ stages: -task VisualStudio.BuildIbcTrainingSettings /p:VisualStudioDropName=$(VisualStudio.DropName) /p:BootstrapperInfoPath=$(Build.StagingDirectory)\MicroBuild\Output\BootstrapperInfo.json + /p:VisualStudioIbcTrainingSettingsPath=$(Build.SourcesDirectory)\eng\config\OptProf.runsettings displayName: 'OptProf - Build IBC training settings' condition: succeeded() diff --git a/eng/config/OptProf.runsettings b/eng/config/OptProf.runsettings new file mode 100644 index 00000000000..3c5f7566a46 --- /dev/null +++ b/eng/config/OptProf.runsettings @@ -0,0 +1,92 @@ + + + + C:\Test\Results + X86 + 1 + 10 + 21600000 + False + False + False + False + False + .NETFramework,Version=v4.0 + STA + %SystemDrive%\Test + False + + + + + + + + + C:\Test\Dumps + + + + + + true + + + + Warning + + + + + + + + + + + + + + + C:\OptProf + C:\Profiles + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 25bfc0867f0efaebcc9ba0cfe0c1f03e0ea54852 Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Thu, 1 Dec 2022 13:39:19 -0600 Subject: [PATCH 8/9] Updating 'Microsoft.DotNet.Arcade.Sdk': '5.0.0-beta.22276.2' => '5.0.0-beta.22526.12' --- eng/Version.Details.xml | 4 +- eng/Versions.props | 3 +- eng/common/build.ps1 | 5 + eng/common/darc-init.ps1 | 3 +- eng/common/init-tools-native.ps1 | 144 ++++++++++++------ eng/common/sdl/packages.config | 2 +- eng/common/templates/job/execute-sdl.yml | 2 +- .../templates/post-build/post-build.yml | 9 +- eng/common/tools.ps1 | 4 + global.json | 2 +- 10 files changed, 124 insertions(+), 54 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 31eb326d23f..d1aa55e64dc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 9c6a04ea1e79e9fcd4e60abd5d2c577075787f93 + 7fafb6feb8f17f5dac9e8930c37016d250032c55 https://github.com/nuget/nuget.client diff --git a/eng/Versions.props b/eng/Versions.props index 8fc95605c2c..5a445f6014b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -2,7 +2,8 @@ - 16.11.3release + 16.11.3 + release 15.1.0.0 preview true diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 94a91c0817e..678e9b20eb7 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -25,6 +25,7 @@ Param( [switch] $prepareMachine, [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '', + [switch] $nativeToolsOnMachine, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -65,6 +66,7 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." @@ -144,6 +146,9 @@ try { $nodeReuse = $false } + if ($nativeToolsOnMachine) { + $env:NativeToolsOnMachine = $true + } if ($restore) { InitializeNativeTools } diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 435e7641341..7df4726cb26 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -10,8 +10,7 @@ param ( function InstallDarcCli ($darcVersion, $toolpath) { $darcCliPackageName = 'microsoft.dotnet.darc' - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" + $dotnet = "dotnet" $toolList = & "$dotnet" tool list -g if ($toolList -like "*$darcCliPackageName*") { diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index db830c00a6f..fbc67effc36 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -31,6 +31,10 @@ Wait time between retry attempts in seconds .PARAMETER GlobalJsonFile File path to global.json file +.PARAMETER PathPromotion +Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) +or break the build if a native tool is not found on the path (on a local dev machine) + .NOTES #> [CmdletBinding(PositionalBinding=$false)] @@ -41,7 +45,8 @@ Param ( [switch] $Force = $False, [int] $DownloadRetries = 5, [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile + [string] $GlobalJsonFile, + [switch] $PathPromotion ) if (!$GlobalJsonFile) { @@ -77,53 +82,102 @@ try { ConvertFrom-Json | Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false + if ($PathPromotion -eq $True) { + if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $InstalledTools = @{} + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + if ($ToolVersion -eq "latest") { + $ToolVersion = "" + } + $ArcadeToolsDirectory = "C:\arcade-tools" + if (-not (Test-Path $ArcadeToolsDirectory)) { + Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." + exit 1 } - if ($showNativeToolsWarning) { - Write-Warning $errMsg + $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) + if ($ToolDirectories -eq $null) { + Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." + exit 1 } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 + $ToolDirectory = $ToolDirectories[0] + $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" + if (-not (Test-Path -Path "$BinPathFile")) { + Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." + exit 1 + } + $BinPath = Get-Content "$BinPathFile" + $ToolPath = Convert-Path -Path $BinPath + Write-Host "Adding $ToolName to the path ($ToolPath)..." + Write-Host "##vso[task.prependpath]$ToolPath" + $env:PATH = "$ToolPath;$env:PATH" + $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } + } + } + return $InstalledTools + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + } } + exit 0 + } + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 } } else { @@ -139,7 +193,7 @@ try { Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" return $InstallBin } - else { + elseif (-not ($PathPromotion)) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' exit 1 } diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index b7bcfe38caf..2cb42e3e7ba 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index ccd2a340934..54775f6a459 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -83,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.110.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.130.0 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index c3ccd64dd37..aaeacddce1e 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -259,7 +259,14 @@ stages: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] pool: - vmImage: 'windows-2019' + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ else }}: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-windows-2022 steps: - task: PowerShell@2 displayName: Publish Using Darc diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 15b2ef4cec8..a0f13b99c02 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -604,6 +604,10 @@ function InitializeNativeTools() { InstallDirectory = "$ToolsDir" } } + if ($env:NativeToolsOnMachine) { + Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..." + $nativeArgs += @{ PathPromotion = $true } + } & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs } } diff --git a/global.json b/global.json index cf3f2800c86..75e55e50687 100644 --- a/global.json +++ b/global.json @@ -12,6 +12,6 @@ }, "msbuild-sdks": { "Microsoft.Build.CentralPackageVersions": "2.0.1", - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.22276.2" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.22526.12" } } From 4a3a06f6542a37031f8691bd7b24d4c2c244c610 Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Thu, 1 Dec 2022 11:56:34 -0600 Subject: [PATCH 9/9] Move BAR publish to windows-latest The win2016 images aren't live any more. --- .vsts-dotnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-dotnet.yml b/.vsts-dotnet.yml index d76ea236266..22315da3e0b 100644 --- a/.vsts-dotnet.yml +++ b/.vsts-dotnet.yml @@ -231,7 +231,7 @@ stages: dependsOn: - Windows_NT pool: - vmImage: vs2017-win2016 + vmImage: windows-latest - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: eng\common\templates\post-build\post-build.yml