microsoft · Haplois · Jul 15, 2021 · Jul 15, 2021
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -15,25 +15,25 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21363.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21364.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
+      <Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21363.2">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21364.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
+      <Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21363.2">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21364.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
+      <Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21363.2">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21364.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
+      <Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21363.2">
+    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21364.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>1b053babb8a542e3ab20f93b3d0aedc732b7e0c7</Sha>
+      <Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
     </Dependency>
     <Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.20258.6">
       <Uri>https://github.com/dotnet/arcade-services</Uri>

diff --git a/eng/Versions.props b/eng/Versions.props
@@ -97,8 +97,8 @@
     <XUnitVersion>2.4.1</XUnitVersion>
     <XUnitAbstractionsVersion>2.0.3</XUnitAbstractionsVersion>
     <XUnitVSRunnerVersion>2.4.1</XUnitVSRunnerVersion>
-    <MicrosoftDotNetBuildTasksFeedVersion>6.0.0-beta.21363.2</MicrosoftDotNetBuildTasksFeedVersion>
-    <MicrosoftDotNetSignToolVersion>6.0.0-beta.21363.2</MicrosoftDotNetSignToolVersion>
+    <MicrosoftDotNetBuildTasksFeedVersion>6.0.0-beta.21364.3</MicrosoftDotNetBuildTasksFeedVersion>
+    <MicrosoftDotNetSignToolVersion>6.0.0-beta.21364.3</MicrosoftDotNetSignToolVersion>
     <MicrosoftAzureDocumentDBVersion>1.22.0</MicrosoftAzureDocumentDBVersion>
     <MicrosoftAzureCosmosDBTableVersion>1.1.2</MicrosoftAzureCosmosDBTableVersion>
     <MicrosoftAspNetCoreAllVersion>2.0.0</MicrosoftAspNetCoreAllVersion>
@@ -108,7 +108,7 @@
     <MicrosoftDotNetMaestroClientVersion>1.1.0-beta.20258.6</MicrosoftDotNetMaestroClientVersion>
     <MicrosoftSourceLinkGitHubVersion>1.1.0-beta-21228-01</MicrosoftSourceLinkGitHubVersion>
     <MicrosoftSourceLinkAzureReposGitVersion>1.1.0-beta-21228-01</MicrosoftSourceLinkAzureReposGitVersion>
-    <MicrosoftDotNetSwaggerGeneratorMSBuildVersion>6.0.0-beta.21363.2</MicrosoftDotNetSwaggerGeneratorMSBuildVersion>
+    <MicrosoftDotNetSwaggerGeneratorMSBuildVersion>6.0.0-beta.21364.3</MicrosoftDotNetSwaggerGeneratorMSBuildVersion>
     <XliffTasksVersion>1.0.0-beta.21265.1</XliffTasksVersion>
     <MicrosoftDotNetMaestroTasksVersion>1.1.0-beta.21228.1</MicrosoftDotNetMaestroTasksVersion>
     <MicrosoftDotNetXHarnessCLIVersion>1.0.0-prerelease.21264.3</MicrosoftDotNetXHarnessCLIVersion>

diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1
@@ -0,0 +1,109 @@
+Param(
+  [string] $GuardianCliLocation,
+  [string] $WorkingDirectory,
+  [string] $TargetDirectory,
+  [string] $GdnFolder,
+  # The list of Guardian tools to configure. For each object in the array:
+  # - If the item is a [hashtable], it must contain these entries:
+  #   - Name = The tool name as Guardian knows it.
+  #   - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique
+  #     among all tool entries with the same Name.
+  #   - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")'
+  # - If the item is a [string] $v, it is treated as '@{ Name="$v" }'
+  [object[]] $ToolsList,
+  [string] $GuardianLoggerLevel='Standard',
+  # Optional: Additional params to add to any tool using CredScan.
+  [string[]] $CrScanAdditionalRunConfigParams,
+  # Optional: Additional params to add to any tool using PoliCheck.
+  [string[]] $PoliCheckAdditionalRunConfigParams
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$global:LASTEXITCODE = 0
+
+try {
+  # `tools.ps1` checks $ci to perform some actions. Since the SDL
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  . $PSScriptRoot\..\tools.ps1
+
+  # Normalize tools list: all in [hashtable] form with defined values for each key.
+  $ToolsList = $ToolsList |
+    ForEach-Object {
+      if ($_ -is [string]) {
+        $_ = @{ Name = $_ }
+      }
+
+      if (-not ($_['Scenario'])) { $_.Scenario = "" }
+      if (-not ($_['Args'])) { $_.Args = @() }
+      $_
+    }
+
+  Write-Host "List of tools to configure:"
+  $ToolsList | ForEach-Object { $_ | Out-String | Write-Host }
+
+  # We store config files in the r directory of .gdn
+  $gdnConfigPath = Join-Path $GdnFolder 'r'
+  $ValidPath = Test-Path $GuardianCliLocation
+
+  if ($ValidPath -eq $False)
+  {
+    Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
+    ExitWithExitCode 1
+  }
+
+  foreach ($tool in $ToolsList) {
+    # Put together the name and scenario to make a unique key.
+    $toolConfigName = $tool.Name
+    if ($tool.Scenario) {
+      $toolConfigName += "_" + $tool.Scenario
+    }
+
+    Write-Host "=== Configuring $toolConfigName..."
+
+    $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
+
+    # For some tools, add default and automatic args.
+    if ($tool.Name -eq 'credscan') {
+      if ($targetDirectory) {
+        $tool.Args += "TargetDirectory < $TargetDirectory"
+      }
+      $tool.Args += "OutputType < pre"
+      $tool.Args += $CrScanAdditionalRunConfigParams
+    } elseif ($tool.Name -eq 'policheck') {
+      if ($targetDirectory) {
+        $tool.Args += "Target < $TargetDirectory"
+      }
+      $tool.Args += $PoliCheckAdditionalRunConfigParams
+    }
+
+    # Create variable pointing to the args array directly so we can use splat syntax later.
+    $toolArgs = $tool.Args
+
+    # Configure the tool. If args array is provided or the current tool has some default arguments
+    # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}",
+    # one per parameter. Doc page for "guardian configure":
+    # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure
+    Exec-BlockVerbosely {
+      & $GuardianCliLocation configure `
+        --working-directory $WorkingDirectory `
+        --tool $tool.Name `
+        --output-path $gdnConfigFile `
+        --logger-level $GuardianLoggerLevel `
+        --noninteractive `
+        --force `
+        $(if ($toolArgs) { "--args" }) @toolArgs
+      Exit-IfNZEC "Sdl"
+    }
+
+    Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile"
+  }
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
+}
diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1
@@ -7,8 +7,17 @@ Param(
   [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,                                         # Required: the directory where source files are located
   [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')),  # Required: the directory where build artifacts are located
   [string] $AzureDevOpsAccessToken,                                                              # Required: access token for dnceng; should be provided via KeyVault
-  [string[]] $SourceToolsList,                                                                   # Optional: list of SDL tools to run on source code
-  [string[]] $ArtifactToolsList,                                                                 # Optional: list of SDL tools to run on built artifacts
+
+  # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
+  # format.
+  [object[]] $SourceToolsList,
+  # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools
+  # list format.
+  [object[]] $ArtifactToolsList,
+  # Optional: list of SDL tools to run without automatically specifying a target directory. See
+  # 'configure-sdl-tool.ps1' for tools list format.
+  [object[]] $CustomToolsList,
+
   [bool] $TsaPublish=$False,                                                                     # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
   [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH,                                               # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
   [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME,                                        # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
@@ -63,13 +72,16 @@ try {
     ExitWithExitCode 1
   }
 
-  & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+  Exec-BlockVerbosely {
+    & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+  }
   $gdnFolder = Join-Path $workingDirectory '.gdn'
 
   if ($TsaOnboard) {
     if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
-      Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
-      & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+      Exec-BlockVerbosely {
+        & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+      }
       if ($LASTEXITCODE -ne 0) {
         Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
         ExitWithExitCode $LASTEXITCODE
@@ -80,20 +92,51 @@ try {
     }
   }
 
-  if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
-    & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+  # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory.
+  function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) {
+    if ($tools -and $tools.Count -gt 0) {
+      Exec-BlockVerbosely {
+        & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') `
+          -GuardianCliLocation $guardianCliLocation `
+          -WorkingDirectory $workingDirectory `
+          -TargetDirectory $targetDirectory `
+          -GdnFolder $gdnFolder `
+          -ToolsList $tools `
+          -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
+          -GuardianLoggerLevel $GuardianLoggerLevel `
+          -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
+          -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+        if ($BreakOnFailure) {
+          Exit-IfNZEC "Sdl"
+        }
+      }
+    }
   }
-  if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
-    & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+
+  # Configure Artifact and Source tools with default Target directories.
+  Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory
+  Configure-ToolsList $SourceToolsList $SourceDirectory
+  # Configure custom tools with no default Target directory.
+  Configure-ToolsList $CustomToolsList $null
+
+  # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call.
+  # (If we used "run" multiple times, each run would overwrite data from earlier runs.)
+  Exec-BlockVerbosely {
+    & $(Join-Path $PSScriptRoot 'run-sdl.ps1') `
+      -GuardianCliLocation $guardianCliLocation `
+      -WorkingDirectory $workingDirectory `
+      -UpdateBaseline $UpdateBaseline `
+      -GdnFolder $gdnFolder
   }
 
   if ($TsaPublish) {
     if ($TsaBranchName -and $BuildNumber) {
       if (-not $TsaRepositoryName) {
         $TsaRepositoryName = "$($Repository)-$($BranchName)"
       }
-      Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
-      & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory  --logger-level $GuardianLoggerLevel
+      Exec-BlockVerbosely {
+        & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory  --logger-level $GuardianLoggerLevel
+      }
       if ($LASTEXITCODE -ne 0) {
         Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
         ExitWithExitCode $LASTEXITCODE
@@ -106,7 +149,11 @@ try {
 
   if ($BreakOnFailure) {
     Write-Host "Failing the build in case of breaking results..."
-    & $guardianCliLocation break
+    Exec-BlockVerbosely {
+      & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+    }
+  } else {
+    Write-Host "Letting the build pass even if there were breaking results..."
   }
 }
 catch {

diff --git a/eng/common/sdl/extract-artifact-archives.ps1 b/eng/common/sdl/extract-artifact-archives.ps1
@@ -0,0 +1,63 @@
+# This script looks for each archive file in a directory and extracts it into the target directory.
+# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**".
+# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip.
+param(
+  # Full path to directory where archives are stored.
+  [Parameter(Mandatory=$true)][string] $InputPath,
+  # Full path to directory to extract archives into. May be the same as $InputPath.
+  [Parameter(Mandatory=$true)][string] $ExtractPath
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+$disableConfigureToolsetImport = $true
+
+try {
+  # `tools.ps1` checks $ci to perform some actions. Since the SDL
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  . $PSScriptRoot\..\tools.ps1
+
+  Measure-Command {
+    $jobs = @()
+
+    # Find archive files for non-Windows and Windows builds.
+    $archiveFiles = @(
+      Get-ChildItem (Join-Path $InputPath "*.tar.gz")
+      Get-ChildItem (Join-Path $InputPath "*.zip")
+    )
+
+    foreach ($targzFile in $archiveFiles) {
+      $jobs += Start-Job -ScriptBlock {
+        $file = $using:targzFile
+        $fileName = [System.IO.Path]::GetFileName($file)
+        $extractDir = Join-Path $using:ExtractPath "$fileName.extracted"
+
+        New-Item $extractDir -ItemType Directory -Force | Out-Null
+
+        Write-Host "Extracting '$file' to '$extractDir'..."
+
+        # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early.
+        # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the
+        # error. Save output so it can be stored in the exception string along with context.
+        $output = tar -xf $file -C $extractDir 2>&1
+        # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we
+        # don't have access to the outer scope.
+        if ($LASTEXITCODE -ne 0) {
+          throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'"
+        }
+
+        Write-Host "Extracted to $extractDir"
+      }
+    }
+
+    Receive-Job $jobs -Wait
+  }
+}
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
+}