Merge remote-tracking branch 'dotnet/features/NullableReferenceTypes'…

… into unconstrained-type-parameter-nullability

* dotnet/features/NullableReferenceTypes:
  Include nullability in CheckConstraints (dotnet#28959)
  Use struct for TypeSymbolWithAnnotations (dotnet#28943)
  Publish Linux Test Results
  Force retest
  Migrate Linux tests to VSTS
  Port determinism fixes
  Make sure TLS 1.2 is used to fetch from https://dot.net
  Move to language version 8
  Make mutex creation more robust
  Disable icacls use on Helix
  Disable leak detection tests on x64
  VSTS YAML file

.vsts-dotnet.yml

@@ -0,0 +1,129 @@
+phases:
+- phase: Windows_Desktop_Unit_Tests
+  queue:
+    name: Helix
+    timeoutInMinutes: 90
+    parallel: 4
+    matrix:
+      debug_32:
+        _configuration: Debug
+        _testKind: Test32
+      debug_64:
+        _configuration: Debug
+        _testKind: Test64
+      release_32:
+        _configuration: Release
+        _testKind: Test32
+      release_64:
+        _configuration: Release
+        _testKind: Test64
+
+  steps:
+    - script: build/scripts/cibuild.cmd -$(_configuration) -testDesktop -$(_testKind)
+
+    - task: PublishTestResults@1
+      inputs:
+        testRunner: XUnit
+        testResultsFiles: '**/xUnitResults/*.xml'
+        mergeTestResults: true
+        testRunTitle: 'Windows Desktop $(_configuration) $(_testKind)'
+      condition: succeededOrFailed()
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)\Binaries\$(_configuration)\Logs'
+        ArtifactName: 'Windows Desktop $(_configuration) $(_testKind)' 
+        publishLocation: Container
+      continueOnError: true
+      condition: failed()
+
+- phase: Windows_CoreClr_Unit_Tests
+  queue:
+    name: Helix
+    timeoutInMinutes: 90
+    parallel: 2
+    matrix:
+      debug:
+        _configuration: Debug
+      release:
+        _configuration: Release
+
+  steps:
+    - script: build/scripts/cibuild.cmd -$(_configuration) -testCoreClr -buildCoreClr
+
+    - task: PublishTestResults@1
+      inputs:
+        testRunner: XUnit
+        testResultsFiles: '**/xUnitResults/*.xml'
+        mergeTestResults: true
+        testRunTitle: 'Windows CoreClr $(_configuration)'
+      condition: succeededOrFailed()
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)\Binaries\$(_configuration)\Logs'
+        ArtifactName: 'Windows CoreClr $(_configuration)'
+        publishLocation: Container
+      continueOnError: true
+      condition: failed()
+
+- phase: Windows_Determinism_Test
+  queue:
+    name: Helix
+    timeoutInMinutes: 90
+  steps:
+    - script: build/scripts/cibuild.cmd -testDeterminism
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)\Binaries\Debug\Logs'
+        ArtifactName: 'Build Determinism Files'
+        publishLocation: Container
+      continueOnError: true
+      condition: failed()
+
+- phase: Windows_Correctness_Test
+  queue:
+    name: Helix
+    timeoutInMinutes: 90
+  steps:
+    - script: build/scripts/test-build-correctness.cmd -cibuild -release
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)\Binaries\Release\Logs'
+        ArtifactName: 'Build Correctness Files'
+        publishLocation: Container
+      continueOnError: true
+      condition: failed()
+
+- phase: Linux_Test
+  queue:
+    name: DotNetCore-Linux                     
+    timeoutInMinutes: 90
+    parallel: 2
+    matrix:
+      coreclr:
+        _args: --debug
+        _name: CoreClr 
+      mono:
+        _args: --debug --docker --mono
+        _name: Mono
+  steps:
+    - script: ./build/scripts/cibuild.sh $(_args)
+    - task: PublishTestResults@1
+      inputs:
+        testRunner: XUnit
+        testResultsFiles: '**/xUnitResults/*.xml'
+        mergeTestResults: true
+        testRunTitle: 'Linux $(_name)'
+      condition: succeededOrFailed()
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)/Binaries/$(_configuration)/Logs'
+        ArtifactName: 'Linux $(_name)'
+        publishLocation: Container
+      continueOnError: true
+      condition: failed()
+

build/Targets/Settings.props

@@ -24,8 +24,11 @@
 
     <JenkinsBuild>false</JenkinsBuild>
     <JenkinsBuild Condition="'$(JENKINS_HOME)' != ''">true</JenkinsBuild>
+    <VstsBuild>false</VstsBuild>
+    <VstsBuild Condition="'$(BUILD_REPOSITORY_ID)' != ''">true</VstsBuild>
     <DeveloperBuild>false</DeveloperBuild>
-    <DeveloperBuild Condition="'$(OfficialBuild)' != 'true' AND '$(JenkinsBuild)' != 'true'">true</DeveloperBuild>
+    <DeveloperBuild Condition="'$(OfficialBuild)' != 'true' AND '$(JenkinsBuild)' != 'true' AND '$(VstsBuild)' != 'true'">true</DeveloperBuild>
+    <ContinuousIntegrationBuild>false</ContinuousIntegrationBuild>
     <ContinuousIntegrationBuild Condition="!$(DeveloperBuild)">true</ContinuousIntegrationBuild>
 
     <NuGetToolPath>$(RepoRoot)nuget.exe</NuGetToolPath>

build/scripts/build-utils.ps1

@@ -148,6 +148,7 @@ function Ensure-DotnetSdk() {
         Create-Directory $cliDir
         Create-Directory $toolsDir
         $destFile = Join-Path $toolsDir "dotnet-install.ps1"
+        [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
         $webClient = New-Object -TypeName "System.Net.WebClient"
         $webClient.DownloadFile("https://dot.net/v1/dotnet-install.ps1", $destFile)
         Exec-Block { & $destFile -Version $sdkVersion -InstallDir $cliDir } | Out-Null

build/scripts/build.ps1

@@ -367,7 +367,7 @@ function Build-InsertionItems() {
 }
 
 function Build-Installer () {
-    #  Copying Artifacts
+    ## Copying Artifacts
     $installerDir = Join-Path $configDir "Installer"
     Create-Directory $installerDir
 

build/scripts/test-determinism.ps1

@@ -1,6 +1,15 @@
+#### 
+#
+#   $bootstrapDir: directory containing the bootstrap compiler
+#   $release: whether to build a debug or release build
+#   $altRootDrive: the drive we build on (via subst) for verifying pathmap implementation
+#
+####
 [CmdletBinding(PositionalBinding=$false)]
 param ( [string]$bootstrapDir = "",
-        [switch]$debugDeterminism = $false)
+        [switch]$release = $false,
+        [string]$altRootDrive = "q:")
+
 
 Set-StrictMode -version 2.0
 $ErrorActionPreference = "Stop"
@@ -12,26 +21,25 @@ $ErrorActionPreference = "Stop"
 $script:skipList = @()
 
 # Location that deterministic error information should be written to. 
-[string]$script:errorDir = ""
-[string]$script:errorDirLeft = ""
-[string]$script:errorDirRight = ""
+[string]$errorDir = ""
+[string]$errorDirLeft = ""
+[string]$errorDirRight = ""
 
-function Run-Build([string]$rootDir, [switch]$restore = $false, [string]$logFile = $null) {
+function Run-Build([string]$rootDir, [string]$logFile = $null) {
     Push-Location $rootDir
     try {
 
         # Clean out the previous run
         Write-Host "Cleaning the Binaries"
-        Exec-Console $msbuild "/nologo /v:m /nodeReuse:false /t:clean Roslyn.sln" 
+        Remove-Item -Recurse (Get-ConfigDir $rootDir) 
+        Remove-Item -Recurse (Get-ObjDir $rootDir) 
 
-        if ($restore) {
-            Write-Host "Restoring the packages"
-            Restore-Project $dotnet "Roslyn.sln"
-        }
+        Write-Host "Restoring the packages"
+        Restore-Project $dotnet "Roslyn.sln"
 
-        $args = "/nologo /v:m /nodeReuse:false /m /p:DebugDeterminism=true /p:BootstrapBuildPath=$script:bootstrapDir /p:Features=`"debug-determinism`" /p:UseRoslynAnalyzers=false /p:DeployExtension=false Roslyn.sln"
+        $args = "/nologo /v:m /nodeReuse:false /m /p:DebugDeterminism=true /p:DeveloperBuild=false /p:BootstrapBuildPath=$script:bootstrapDir /p:Features=`"debug-determinism`" /p:UseRoslynAnalyzers=false /p:DeployExtension=false Roslyn.sln"
         if ($logFile -ne $null) {
-            $logFile = Join-Path $binariesDir $logFile
+            $logFile = Join-Path $logDir $logFile
             $args += " /bl:$logFile"
         }
 
@@ -47,27 +55,45 @@ function Get-ObjDir([string]$rootDir) {
     return Join-Path $rootDir "Binaries\Obj"
 }
 
+function Get-ConfigDir([string]$rootDir) { 
+    return Join-Path $rootDir "Binaries\$buildConfiguration"
+}
+
 # Return all of the files that need to be processed for determinism under the given
 # directory.
 function Get-FilesToProcess([string]$rootDir) {
     $objDir = Get-ObjDir $rootDir
-    foreach ($item in Get-ChildItem -re -in *.dll,*.exe,*.pdb $objDir) {
-        $fileFullName = $item.FullName 
-        $fileName = Split-Path -leaf $fileFullName
+    foreach ($item in Get-ChildItem -re -in *.dll,*.exe,*.pdb,*.sourcelink.json $objDir) {
+        $filePath = $item.FullName 
+        $fileName = Split-Path -leaf $filePath
 
         if ($skipList.Contains($fileName)) {
             continue;
         }
 
-        $fileId = $fileFullName.Substring($objDir.Length).Replace("\", ".")
-        $fileHash = (Get-FileHash $fileFullName -algorithm MD5).Hash
+        $fileId = $filePath.Substring($objDir.Length).Replace("\", ".")
+        $fileHash = (Get-FileHash $filePath -algorithm MD5).Hash
 
         $data = @{}
         $data.Hash = $fileHash
-        $data.Content = [IO.File]::ReadAllBytes($fileFullName)
+        $data.Content = [IO.File]::ReadAllBytes($filePath)
         $data.FileId = $fileId
         $data.FileName = $fileName
-        $data.FileFullName = $fileFullName
+        $data.FilePath = $filePath
+
+        $keyFilePath = $filePath + ".key"
+        $keyFileName = Split-Path -leaf $keyFilePath
+        if (Test-Path $keyFilePath) { 
+            $data.KeyFileName = $keyFileName
+            $data.KeyFilePath = $keyFilePath
+            $data.KeyFileContent = [IO.File]::ReadAllBytes($keyFilePath)
+        }
+        else {
+            $data.KeyFileName = ""
+            $data.KeyFilePath = ""
+            $data.KeyFileContent = $null
+        }
+
         Write-Output $data
     }
 }
@@ -117,8 +143,8 @@ function Test-MapContents($dataMap) {
     }
 }
 
-function Test-Build([string]$rootDir, $dataMap, [string]$logFile, [switch]$restore = $false) {
-    Run-Build $rootDir -logFile $logFile -restore:$restore
+function Test-Build([string]$rootDir, $dataMap, [string]$logFile) {
+    Run-Build $rootDir -logFile $logFile
 
     $errorList = @()
     $allGood = $true
@@ -127,10 +153,10 @@ function Test-Build([string]$rootDir, $dataMap, [string]$logFile, [switch]$resto
     foreach ($fileData in Get-FilesToProcess $rootDir) {
         $fileId = $fileData.FileId
         $fileName = $fileData.FileName
-        $fileFullName = $fileData.FileFullName
+        $filePath = $fileData.FilePath
 
         if (-not $dataMap.Contains($fileId)) {
-            Write-Host "ERROR! Missing entry in map $fileId->$fileFullName"
+            Write-Host "ERROR! Missing entry in map $fileId->$filePath"
             $allGood = $false
             continue
         }
@@ -142,8 +168,16 @@ function Test-Build([string]$rootDir, $dataMap, [string]$logFile, [switch]$resto
             $errorList += $fileName
 
             # Save out the original and baseline so Jenkins will archive them for investigation
-            [IO.File]::WriteAllBytes((Join-Path $script:errorDirLeft $fileName), $oldFileData.Content)
-            Copy-Item $fileFullName (Join-Path $script:errorDirRight $fileName)
+            [IO.File]::WriteAllBytes((Join-Path $errorDirLeft $fileName), $oldFileData.Content)
+            Copy-Item $filePath (Join-Path $errorDirRight $fileName)
+
+            # Copy the key files if available too
+            $keyFileName = $oldFileData.KeyFileName
+            if ($keyFileName -ne "") {
+                [IO.File]::WriteAllBytes((Join-Path $errorDirLeft $keyFileName), $oldFileData.KeyFileContent)
+                Copy-Item $fileData.KeyFilePath (Join-Path $errorDirRight $keyFileName)
+            }
+
             continue
         }
 
@@ -157,7 +191,7 @@ function Test-Build([string]$rootDir, $dataMap, [string]$logFile, [switch]$resto
         }
 
         Write-Host "Archiving failure information"
-        $zipFile = Join-Path $repoDir "Binaries\determinism.zip"
+        $zipFile = Join-Path $logDir "determinism.zip"
         Add-Type -Assembly "System.IO.Compression.FileSystem";
         [System.IO.Compression.ZipFile]::CreateFromDirectory($script:errorDir, $zipFile, "Fastest", $true);
 
@@ -169,14 +203,6 @@ function Test-Build([string]$rootDir, $dataMap, [string]$logFile, [switch]$resto
 function Run-Test() {
     $rootDir = $repoDir
 
-    # Ensure the error directory is written for all analysis to use.
-    $script:errorDir = Join-Path $repoDir "Binaries\Determinism"
-    $script:errorDirLeft = Join-Path $script:errorDir "Left"
-    $script:errorDirRight = Join-Path $script:errorDir "Right"
-    Create-Directory $script:errorDir
-    Create-Directory $script:errorDirLeft
-    Create-Directory $script:errorDirRight
-
     # Run the initial build so that we can populate the maps
     Run-Build $repoDir -logFile "initial.binlog"
     $dataMap = Record-Binaries $repoDir
@@ -188,19 +214,30 @@ function Run-Test() {
     # Run another build in a different source location and verify that path mapping 
     # allows the build to be identical.  To do this we'll copy the entire source 
     # tree under the Binaries\q directory and run a build from there.
-    $altRootDir = Join-Path "$repoDir\Binaries" "q"
-    Remove-Item -re -fo $altRootDir -ErrorAction SilentlyContinue
-    & robocopy $repoDir $altRootDir /E /XD $binariesDir /XD ".git" /njh /njs /ndl /nc /ns /np /nfl
-
-    # Symlink the .git directory to make SourceLink think Binaries/q is the repo root:
-    & cmd /c mklink /d (Join-Path $altRootDir ".git") (Join-Path $repoDir ".git")
-
-    Test-Build -rootDir $altRootDir -dataMap $dataMap -logFile "test2.binlog" -restore
+    Write-Host "Building in a different directory"
+    Exec-Command "subst" "$altRootDrive $(Split-Path -parent $repoDir)"
+    try {
+        $altRootDir = Join-Path "$($altRootDrive)\" (Split-Path -leaf $repoDir)
+        Test-Build -rootDir $altRootDir -dataMap $dataMap -logFile "test2.binlog"
+    }
+    finally {
+        Exec-Command "subst" "$altRootDrive /d"
+    }
 }
 
 try {
     . (Join-Path $PSScriptRoot "build-utils.ps1")
 
+    # Create all of the logging directories
+    $buildConfiguration = if ($release) { "Release" } else { "Debug" }
+    $logDir = Join-Path (Get-ConfigDir $repoDir) "Logs"
+    $errorDir = Join-Path $binariesDir "Determinism"
+    $errorDirLeft = Join-Path $errorDir "Left"
+    $errorDirRight = Join-Path $errorDir "Right"
+    Create-Directory $logDir
+    Create-Directory $errorDirLeft
+    Create-Directory $errorDirRight
+
     $dotnet = Ensure-DotnetSdk
     $msbuild = Ensure-MSBuild
     if (($bootstrapDir -eq "") -or (-not ([IO.Path]::IsPathRooted($script:bootstrapDir)))) {