Experiment, parallelize some tests

azure-pipelines-PR.yml

@@ -488,6 +488,8 @@ stages:
             displayName: Publish Test Results
             inputs:
               testResultsFormat: 'XUnit'
+              testRunTitle: WindowsCompressedMetadata $(_testKind)
+              mergeTestResults: true
               testResultsFiles: '*.xml'
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)'
             continueOnError: true
@@ -558,7 +560,9 @@ stages:
             displayName: Publish Test Results
             inputs:
               testResultsFormat: 'XUnit'
+              testRunTitle: Linux
               testResultsFiles: '*.xml'
+              mergeTestResults: true
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
             continueOnError: true
             condition: always()
@@ -602,6 +606,8 @@ stages:
             inputs:
               testResultsFormat: 'XUnit'
               testResultsFiles: '*.xml'
+              testRunTitle: MacOS
+              mergeTestResults: true
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
             continueOnError: true
             condition: always()

docs/release-notes/.FSharp.Compiler.Service/9.0.200.md

@@ -13,11 +13,13 @@
 ### Changed
 
 * Make ILTypeDef interface impls calculation lazy. ([PR #17392](https://github.com/dotnet/fsharp/pull/17392))
+* Adjustments to better run in parallel testing ([PR #17662](https://github.com/dotnet/fsharp/pull/17662))
 * Remove non-functional useSyntaxTreeCache option. ([PR #17768](https://github.com/dotnet/fsharp/pull/17768))
 * Better ranges for CE `let!` and `use!` error reporting. ([PR #17712](https://github.com/dotnet/fsharp/pull/17712))
 * Better ranges for CE `do!` error reporting. ([PR #17779](https://github.com/dotnet/fsharp/pull/17779))
 * Better ranges for CE `return, yield, return! and yield!` error reporting. ([PR #17792](https://github.com/dotnet/fsharp/pull/17792))
 * Better ranges for CE `match!`. ([PR #17789](https://github.com/dotnet/fsharp/pull/17789))
 * Better ranges for CE `use` error reporting. ([PR #17811](https://github.com/dotnet/fsharp/pull/17811))
 
+
 ### Breaking Changes

eng/Build.ps1

@@ -369,7 +369,7 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
     $testLogPath = "$ArtifactsDir\TestResults\$configuration\${projectName}_$targetFramework.xml"
     $testBinLogPath = "$LogDir\${projectName}_$targetFramework.binlog"
     $args = "test $testProject -c $configuration -f $targetFramework -v n --test-adapter-path $testadapterpath --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath"
-    $args += " --blame --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=false"
+    $args += " --blame --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=false"
 
     if (-not $noVisualStudio -or $norestore) {
         $args += " --no-restore"
@@ -395,6 +395,28 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
     }
 }
 
+function TestSolutionUsingMSBuild([string] $testSolution, [string] $targetFramework, [string] $testadapterpath) {
+    $dotnetPath = InitializeDotNetCli
+    $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
+    $solutionName = [System.IO.Path]::GetFileNameWithoutExtension($testSolution)
+    $testLogPath = "$ArtifactsDir\TestResults\$configuration\{assembly}.{framework}.xml"
+    $testBinLogPath = "$LogDir\${solutionName}_$targetFramework.binlog"
+
+    $args = "test $testSolution -c $configuration -f $targetFramework --test-adapter-path $testadapterpath -v minimal --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath"
+    $args += " --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration /p:VsTestUseMSBuildOutput=true"
+
+    if (-not $noVisualStudio -or $norestore) {
+        $args += " --no-restore"
+    }
+
+    if (-not $noVisualStudio) {
+        $args += " --no-build"
+    }
+
+    Write-Host("$args")
+    Exec-Console $dotnetExe $args
+}
+
 function Prepare-TempDir() {
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir
@@ -589,31 +611,11 @@ try {
     $script:BuildMessage = "Failure running tests"
 
     if ($testCoreClr) {
-        $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true
-
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\"
-
-        # Collect output from  background jobs
-        Wait-job $bgJob | out-null
-        Receive-Job $bgJob -ErrorAction Stop
+        TestSolutionUsingMSBuild -testSolution "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
     }
 
     if ($testDesktop) {
-        $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true
-
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework  -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\"
-
-        # Collect output from  background jobs
-        Wait-job $bgJob | out-null
-        Receive-Job $bgJob -ErrorAction Stop
+        TestSolutionUsingMSBuild -testSolution "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
     }
 
     if ($testFSharpQA) {

eng/Versions.props

@@ -184,7 +184,7 @@
     <MicrosoftNETCoreAppRefVersion>3.1.0</MicrosoftNETCoreAppRefVersion>
     <MicrosoftNETCoreILDAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILDAsmVersion>
     <MicrosoftNETCoreILAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILAsmVersion>
-    <MicrosoftNETTestSdkVersion>17.4.0</MicrosoftNETTestSdkVersion>
+    <MicrosoftNETTestSdkVersion>17.11.1</MicrosoftNETTestSdkVersion>
     <NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
     <RoslynToolsSignToolVersion>1.0.0-beta2-dev3</RoslynToolsSignToolVersion>
     <StreamJsonRpcVersion>2.18.48</StreamJsonRpcVersion>

eng/build.sh

@@ -214,7 +214,7 @@ function Test() {
   projectname=$(basename -- "$testproject")
   projectname="${projectname%.*}"
   testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml"
-  args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
+  args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
   "$DOTNET_INSTALL_DIR/dotnet" $args || exit $?
 }
 

src/Compiler/Driver/fsc.fs

@@ -273,9 +273,6 @@ let SetProcessThreadLocals tcConfigB =
     | Some s -> Thread.CurrentThread.CurrentUICulture <- CultureInfo(s)
     | None -> ()
 
-    if tcConfigB.utf8output then
-        Console.OutputEncoding <- Encoding.UTF8
-
 let ProcessCommandLineFlags (tcConfigB: TcConfigBuilder, lcidFromCodePage, argv) =
     let mutable inputFilesRef = []
 
@@ -550,6 +547,17 @@ let main1
     | Some parallelReferenceResolution -> tcConfigB.parallelReferenceResolution <- parallelReferenceResolution
     | None -> ()
 
+    if tcConfigB.utf8output && Console.OutputEncoding <> Encoding.UTF8 then
+        let previousEncoding = Console.OutputEncoding
+        Console.OutputEncoding <- Encoding.UTF8
+
+        disposables.Register(
+            { new IDisposable with
+                member _.Dispose() =
+                    Console.OutputEncoding <- previousEncoding
+            }
+        )
+
     // Display the banner text, if necessary
     if not bannerAlreadyPrinted then
         Console.Write(GetBannerText tcConfigB)
@@ -1242,16 +1250,6 @@ let CompileFromCommandLineArguments
     ) =
 
     use disposables = new DisposablesTracker()
-    let savedOut = Console.Out
-
-    use _ =
-        { new IDisposable with
-            member _.Dispose() =
-                try
-                    Console.SetOut(savedOut)
-                with _ ->
-                    ()
-        }
 
     main1 (
         ctok,

src/Compiler/Interactive/fsi.fs

@@ -846,7 +846,7 @@ type internal FsiStdinSyphon(errorWriter: TextWriter) =
 /// Encapsulates functions used to write to outWriter and errorWriter
 type internal FsiConsoleOutput(tcConfigB, outWriter: TextWriter, errorWriter: TextWriter) =
 
-    let nullOut = new StreamWriter(Stream.Null) :> TextWriter
+    let nullOut = TextWriter.Null
 
     let fprintfnn (os: TextWriter) fmt =
         Printf.kfprintf
@@ -1203,11 +1203,6 @@ type internal FsiCommandLineOptions(fsi: FsiEvaluationSessionHostConfig, argv: s
         if tcConfigB.clearResultsCache then
             dependencyProvider.ClearResultsCache(tcConfigB.compilerToolPaths, getOutputDir tcConfigB, reportError rangeCmdArgs)
 
-        if tcConfigB.utf8output then
-            let prev = Console.OutputEncoding
-            Console.OutputEncoding <- Encoding.UTF8
-            System.AppDomain.CurrentDomain.ProcessExit.Add(fun _ -> Console.OutputEncoding <- prev)
-
     do
         let firstArg =
             match sourceFiles with
@@ -4646,6 +4641,20 @@ type FsiEvaluationSession
         with e ->
             warning (e)
 
+    let restoreEncoding =
+        if tcConfigB.utf8output && Console.OutputEncoding <> Text.Encoding.UTF8 then
+            let previousEncoding = Console.OutputEncoding
+            Console.OutputEncoding <- Encoding.UTF8
+
+            Some(
+                { new IDisposable with
+                    member _.Dispose() =
+                        Console.OutputEncoding <- previousEncoding
+                }
+            )
+        else
+            None
+
     do
         updateBannerText () // resetting banner text after parsing options
 
@@ -4789,6 +4798,7 @@ type FsiEvaluationSession
         member _.Dispose() =
             (tcImports :> IDisposable).Dispose()
             uninstallMagicAssemblyResolution.Dispose()
+            restoreEncoding |> Option.iter (fun x -> x.Dispose())
 
     /// Load the dummy interaction, load the initial files, and,
     /// if interacting, start the background thread to read the standard input.

src/Compiler/Service/service.fs

@@ -100,14 +100,6 @@ module CompileHelpers =
 
         diagnostics.ToArray(), result
 
-    let setOutputStreams execute =
-        // Set the output streams, if requested
-        match execute with
-        | Some(writer, error) ->
-            Console.SetOut writer
-            Console.SetError error
-        | None -> ()
-
 [<Sealed; AutoSerializable(false)>]
 // There is typically only one instance of this type in an IDE process.
 type FSharpChecker

src/Compiler/Utilities/Activity.fs

@@ -237,38 +237,36 @@ module internal Activity =
 
             sb.ToString()
 
-        let addCsvFileListener (pathToFile:string) =
-            if pathToFile |> File.Exists |> not then
-                File.WriteAllLines(
-                    pathToFile,
-                    [
-                        "Name,StartTime,EndTime,Duration(s),Id,ParentId,RootId,"
-                        + String.concat "," Tags.AllKnownTags
-                    ]
-                )
-
-            let sw = new StreamWriter(path = pathToFile, append = true)
+        let addCsvFileListener (pathToFile: string) =
+            let newFile = pathToFile |> File.Exists |> not
+            // FileShare.Read to avoid sporadic file locking during tests. 
+            let stream = new FileStream(pathToFile, FileMode.Append, FileAccess.Write, FileShare.Read)
+            let csvWriter = new StreamWriter(stream)
+
+            if newFile then csvWriter.WriteLine(
+                "Name,StartTime,EndTime,Duration(s),Id,ParentId,RootId,"
+                + String.concat "," Tags.AllKnownTags)
 
             let msgQueue =
                 MailboxProcessor<string>.Start(fun inbox ->
                     async {
                         while true do
                             let! msg = inbox.Receive()
-                            do! sw.WriteLineAsync(msg) |> Async.AwaitTask
+                            do! csvWriter.WriteLineAsync(msg) |> Async.AwaitTask
                     })
 
-            let l =
+            let listener =
                 new ActivityListener(
                     ShouldListenTo = (fun a ->ActivityNames.AllRelevantNames |> Array.contains a.Name),
                     Sample = (fun _ -> ActivitySamplingResult.AllData),
                     ActivityStopped = (fun a -> msgQueue.Post(createCsvRow a))
                 )
 
-            ActivitySource.AddActivityListener(l)
+            ActivitySource.AddActivityListener(listener)
 
             { new IDisposable with
                 member this.Dispose() =
-                    l.Dispose() // Unregister from listening new activities first
+                    listener.Dispose() // Unregister from listening new activities first
                     (msgQueue :> IDisposable).Dispose() // Wait for the msg queue to be written out
-                    sw.Dispose() // Only then flush the messages and close the file
+                    csvWriter.Dispose() // Only then flush the messages and close the file
             }

src/Compiler/Utilities/illib.fs

@@ -137,7 +137,7 @@ module internal PervasiveAutoOpens =
     type Async with
 
         static member RunImmediate(computation: Async<'T>, ?cancellationToken) =
-            let cancellationToken = defaultArg cancellationToken Async.DefaultCancellationToken
+            let cancellationToken = defaultArg cancellationToken CancellationToken.None
 
             let ts = TaskCompletionSource<'T>()
 

src/fsi/fsimain.fs

@@ -358,16 +358,6 @@ let evaluateSession (argv: string[]) =
 let MainMain argv =
     ignore argv
     let argv = System.Environment.GetCommandLineArgs()
-    let savedOut = Console.Out
-
-    use __ =
-        { new IDisposable with
-            member _.Dispose() =
-                try
-                    Console.SetOut(savedOut)
-                with _ ->
-                    ()
-        }
 
     let timesFlag = argv |> Array.exists (fun x -> x = "/times" || x = "--times")
 

tests/EndToEndBuildTests/BasicProvider/BasicProvider.Tests/BasicProvider.Tests.fsproj

@@ -21,7 +21,7 @@
         <PackagePath>content\myfiles\</PackagePath>
     </Content>
     <PackageReference Include="BasicProvider" Version="1.0.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
   </ItemGroup>
 
   <Target Name="RemovePackagesFromCache" BeforeTargets="Restore">

tests/EndToEndBuildTests/ComboProvider/ComboProvider.Tests/ComboProvider.Tests.fsproj

@@ -18,7 +18,8 @@
     </None>
 
     <PackageReference Include="ComboProvider" Version="1.0.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+
   </ItemGroup>
 
   <Target Name="RemovePackagesFromCache" BeforeTargets="Restore">

tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj

@@ -4,17 +4,26 @@
 
   <PropertyGroup>
     <TargetFrameworks>net472;$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
-    <TargetFrameworks Condition="'$(OS)' == 'Unix'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(OS)' == 'Unix' or '$(BUILDING_USING_DOTNET)' == 'true'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
     <OutputType>Library</OutputType>
     <DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
     <UnitTestType>xunit</UnitTestType>
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
+      <Link>XunitSetup.fs</Link>
+    </Compile>
     <Compile Include="WriteCodeFragmentTests.fs" />
     <Compile Include="MapSourceRootsTests.fs" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="xunit.runner.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content> 
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Build\FSharp.Build.fsproj" />
     <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Core\FSharp.Core.fsproj" />

tests/FSharp.Build.UnitTests/xunit.runner.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "appDomain": "denied",
+  "parallelizeAssembly": true
+}