Skip to content

Commit

Permalink
Update dependencies from https://github.com/dotnet/arcade build 20210…
Browse files Browse the repository at this point in the history
…714.3 (#921)

Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SwaggerGenerator.MSBuild , Microsoft.DotNet.SignTool , Microsoft.DotNet.Helix.Sdk
 From Version 6.0.0-beta.21363.2 -> To Version 6.0.0-beta.21364.3

Co-authored-by: dotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>
  • Loading branch information
dotnet-maestro[bot] and dotnet-maestro[bot] authored Jul 15, 2021
1 parent a0882c1 commit 90112ff
Show file tree
Hide file tree
Showing 9 changed files with 410 additions and 72 deletions.
20 changes: 10 additions & 10 deletions eng/Version.Details.xml
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,25 @@
</Dependency>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21363.2">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21364.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
<Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21363.2">
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21364.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
<Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21363.2">
<Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21364.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
<Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21363.2">
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21364.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
<Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21363.2">
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21364.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
<Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.20258.6">
<Uri>https://github.com/dotnet/arcade-services</Uri>
Expand Down
6 changes: 3 additions & 3 deletions eng/Versions.props
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@
<XUnitVersion>2.4.1</XUnitVersion>
<XUnitAbstractionsVersion>2.0.3</XUnitAbstractionsVersion>
<XUnitVSRunnerVersion>2.4.1</XUnitVSRunnerVersion>
<MicrosoftDotNetBuildTasksFeedVersion>6.0.0-beta.21363.2</MicrosoftDotNetBuildTasksFeedVersion>
<MicrosoftDotNetSignToolVersion>6.0.0-beta.21363.2</MicrosoftDotNetSignToolVersion>
<MicrosoftDotNetBuildTasksFeedVersion>6.0.0-beta.21364.3</MicrosoftDotNetBuildTasksFeedVersion>
<MicrosoftDotNetSignToolVersion>6.0.0-beta.21364.3</MicrosoftDotNetSignToolVersion>
<MicrosoftAzureDocumentDBVersion>1.22.0</MicrosoftAzureDocumentDBVersion>
<MicrosoftAzureCosmosDBTableVersion>1.1.2</MicrosoftAzureCosmosDBTableVersion>
<MicrosoftAspNetCoreAllVersion>2.0.0</MicrosoftAspNetCoreAllVersion>
Expand All @@ -108,7 +108,7 @@
<MicrosoftDotNetMaestroClientVersion>1.1.0-beta.20258.6</MicrosoftDotNetMaestroClientVersion>
<MicrosoftSourceLinkGitHubVersion>1.1.0-beta-21228-01</MicrosoftSourceLinkGitHubVersion>
<MicrosoftSourceLinkAzureReposGitVersion>1.1.0-beta-21228-01</MicrosoftSourceLinkAzureReposGitVersion>
<MicrosoftDotNetSwaggerGeneratorMSBuildVersion>6.0.0-beta.21363.2</MicrosoftDotNetSwaggerGeneratorMSBuildVersion>
<MicrosoftDotNetSwaggerGeneratorMSBuildVersion>6.0.0-beta.21364.3</MicrosoftDotNetSwaggerGeneratorMSBuildVersion>
<XliffTasksVersion>1.0.0-beta.21265.1</XliffTasksVersion>
<MicrosoftDotNetMaestroTasksVersion>1.1.0-beta.21228.1</MicrosoftDotNetMaestroTasksVersion>
<MicrosoftDotNetXHarnessCLIVersion>1.0.0-prerelease.21264.3</MicrosoftDotNetXHarnessCLIVersion>
Expand Down
109 changes: 109 additions & 0 deletions eng/common/sdl/configure-sdl-tool.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
Param(
[string] $GuardianCliLocation,
[string] $WorkingDirectory,
[string] $TargetDirectory,
[string] $GdnFolder,
# The list of Guardian tools to configure. For each object in the array:
# - If the item is a [hashtable], it must contain these entries:
# - Name = The tool name as Guardian knows it.
# - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique
# among all tool entries with the same Name.
# - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")'
# - If the item is a [string] $v, it is treated as '@{ Name="$v" }'
[object[]] $ToolsList,
[string] $GuardianLoggerLevel='Standard',
# Optional: Additional params to add to any tool using CredScan.
[string[]] $CrScanAdditionalRunConfigParams,
# Optional: Additional params to add to any tool using PoliCheck.
[string[]] $PoliCheckAdditionalRunConfigParams
)

$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$global:LASTEXITCODE = 0

try {
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1

# Normalize tools list: all in [hashtable] form with defined values for each key.
$ToolsList = $ToolsList |
ForEach-Object {
if ($_ -is [string]) {
$_ = @{ Name = $_ }
}

if (-not ($_['Scenario'])) { $_.Scenario = "" }
if (-not ($_['Args'])) { $_.Args = @() }
$_
}

Write-Host "List of tools to configure:"
$ToolsList | ForEach-Object { $_ | Out-String | Write-Host }

# We store config files in the r directory of .gdn
$gdnConfigPath = Join-Path $GdnFolder 'r'
$ValidPath = Test-Path $GuardianCliLocation

if ($ValidPath -eq $False)
{
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
ExitWithExitCode 1
}

foreach ($tool in $ToolsList) {
# Put together the name and scenario to make a unique key.
$toolConfigName = $tool.Name
if ($tool.Scenario) {
$toolConfigName += "_" + $tool.Scenario
}

Write-Host "=== Configuring $toolConfigName..."

$gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"

# For some tools, add default and automatic args.
if ($tool.Name -eq 'credscan') {
if ($targetDirectory) {
$tool.Args += "TargetDirectory < $TargetDirectory"
}
$tool.Args += "OutputType < pre"
$tool.Args += $CrScanAdditionalRunConfigParams
} elseif ($tool.Name -eq 'policheck') {
if ($targetDirectory) {
$tool.Args += "Target < $TargetDirectory"
}
$tool.Args += $PoliCheckAdditionalRunConfigParams
}

# Create variable pointing to the args array directly so we can use splat syntax later.
$toolArgs = $tool.Args

# Configure the tool. If args array is provided or the current tool has some default arguments
# defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}",
# one per parameter. Doc page for "guardian configure":
# https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure
Exec-BlockVerbosely {
& $GuardianCliLocation configure `
--working-directory $WorkingDirectory `
--tool $tool.Name `
--output-path $gdnConfigFile `
--logger-level $GuardianLoggerLevel `
--noninteractive `
--force `
$(if ($toolArgs) { "--args" }) @toolArgs
Exit-IfNZEC "Sdl"
}

Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile"
}
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1
}
71 changes: 59 additions & 12 deletions eng/common/sdl/execute-all-sdl-tools.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,17 @@ Param(
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
[string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located
[string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
[string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code
[string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts

# Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
# format.
[object[]] $SourceToolsList,
# Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools
# list format.
[object[]] $ArtifactToolsList,
# Optional: list of SDL tools to run without automatically specifying a target directory. See
# 'configure-sdl-tool.ps1' for tools list format.
[object[]] $CustomToolsList,

[bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
[string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
[string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
Expand Down Expand Up @@ -63,13 +72,16 @@ try {
ExitWithExitCode 1
}

& $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
Exec-BlockVerbosely {
& $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
}
$gdnFolder = Join-Path $workingDirectory '.gdn'

if ($TsaOnboard) {
if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
& $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
Exec-BlockVerbosely {
& $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
}
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
Expand All @@ -80,20 +92,51 @@ try {
}
}

if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
# Configure a list of tools with a default target directory. Populates the ".gdn/r" directory.
function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) {
if ($tools -and $tools.Count -gt 0) {
Exec-BlockVerbosely {
& $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') `
-GuardianCliLocation $guardianCliLocation `
-WorkingDirectory $workingDirectory `
-TargetDirectory $targetDirectory `
-GdnFolder $gdnFolder `
-ToolsList $tools `
-AzureDevOpsAccessToken $AzureDevOpsAccessToken `
-GuardianLoggerLevel $GuardianLoggerLevel `
-CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
-PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
if ($BreakOnFailure) {
Exit-IfNZEC "Sdl"
}
}
}
}
if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams

# Configure Artifact and Source tools with default Target directories.
Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory
Configure-ToolsList $SourceToolsList $SourceDirectory
# Configure custom tools with no default Target directory.
Configure-ToolsList $CustomToolsList $null

# At this point, all tools are configured in the ".gdn" directory. Run them all in a single call.
# (If we used "run" multiple times, each run would overwrite data from earlier runs.)
Exec-BlockVerbosely {
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') `
-GuardianCliLocation $guardianCliLocation `
-WorkingDirectory $workingDirectory `
-UpdateBaseline $UpdateBaseline `
-GdnFolder $gdnFolder
}

if ($TsaPublish) {
if ($TsaBranchName -and $BuildNumber) {
if (-not $TsaRepositoryName) {
$TsaRepositoryName = "$($Repository)-$($BranchName)"
}
Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
& $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
Exec-BlockVerbosely {
& $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
}
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
Expand All @@ -106,7 +149,11 @@ try {

if ($BreakOnFailure) {
Write-Host "Failing the build in case of breaking results..."
& $guardianCliLocation break
Exec-BlockVerbosely {
& $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
}
} else {
Write-Host "Letting the build pass even if there were breaking results..."
}
}
catch {
Expand Down
63 changes: 63 additions & 0 deletions eng/common/sdl/extract-artifact-archives.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# This script looks for each archive file in a directory and extracts it into the target directory.
# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**".
# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip.
param(
# Full path to directory where archives are stored.
[Parameter(Mandatory=$true)][string] $InputPath,
# Full path to directory to extract archives into. May be the same as $InputPath.
[Parameter(Mandatory=$true)][string] $ExtractPath
)

$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0

$disableConfigureToolsetImport = $true

try {
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1

Measure-Command {
$jobs = @()

# Find archive files for non-Windows and Windows builds.
$archiveFiles = @(
Get-ChildItem (Join-Path $InputPath "*.tar.gz")
Get-ChildItem (Join-Path $InputPath "*.zip")
)

foreach ($targzFile in $archiveFiles) {
$jobs += Start-Job -ScriptBlock {
$file = $using:targzFile
$fileName = [System.IO.Path]::GetFileName($file)
$extractDir = Join-Path $using:ExtractPath "$fileName.extracted"

New-Item $extractDir -ItemType Directory -Force | Out-Null

Write-Host "Extracting '$file' to '$extractDir'..."

# Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early.
# This type of quit skips the catch, so we wouldn't be able to tell which file triggered the
# error. Save output so it can be stored in the exception string along with context.
$output = tar -xf $file -C $extractDir 2>&1
# Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we
# don't have access to the outer scope.
if ($LASTEXITCODE -ne 0) {
throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'"
}

Write-Host "Extracted to $extractDir"
}
}

Receive-Job $jobs -Wait
}
}
catch {
Write-Host $_
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1
}
Loading

0 comments on commit 90112ff

Please sign in to comment.