From cef21364307b0d8c59538dd281751fd409fd0f9d Mon Sep 17 00:00:00 2001 From: Anh-Dung Phan Date: Mon, 26 Aug 2019 08:21:07 +0100 Subject: [PATCH 1/4] Support VSTest parallel test execution setting --- src/app/Fake.DotNet.Testing.VSTest/VSTest.fs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs b/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs index 5eba8133af2..1a3505c4f4a 100644 --- a/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs +++ b/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs @@ -30,6 +30,8 @@ type VSTestParams = SettingsPath : string /// Names of the tests that should be run (optional). Tests : seq + /// Enables parallel test execution (optional). + Parallel : bool /// Enables code coverage collection (optional). EnableCodeCoverage : bool /// Run the tests in an isolated process (optional). @@ -69,6 +71,7 @@ type VSTestParams = let private VSTestDefaults = { SettingsPath = null Tests = [] + Parallel = false EnableCodeCoverage = false InIsolation = true UseVsixExtensions = false @@ -100,6 +103,7 @@ let buildArgs (parameters : VSTestParams) (assembly: string) = |> StringBuilder.appendIfTrue (assembly <> null) assembly |> StringBuilder.appendIfNotNull parameters.SettingsPath "/Settings:" |> StringBuilder.appendIfTrue (testsToRun <> null) testsToRun + |> StringBuilder.appendIfTrue parameters.Parallel "/Parallel" |> StringBuilder.appendIfTrue parameters.EnableCodeCoverage "/EnableCodeCoverage" |> StringBuilder.appendIfTrue parameters.InIsolation "/InIsolation" |> StringBuilder.appendIfTrue parameters.UseVsixExtensions "/UseVsixExtensions:true" From 0186b4dfb32688df57d60b10975b4cde91343561 Mon Sep 17 00:00:00 2001 From: Anh-Dung Phan Date: Mon, 26 Aug 2019 15:02:49 +0100 Subject: [PATCH 2/4] Use CreateProcess API --- src/app/Fake.DotNet.Testing.VSTest/VSTest.fs | 58 ++++++++++++-------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs b/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs index 1a3505c4f4a..426d99b50cb 100644 --- a/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs +++ b/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs @@ -5,6 +5,7 @@ module Fake.DotNet.Testing.VSTest open Fake.Core open Fake.Testing.Common open System +open System.IO open System.Text /// [omit] @@ -94,15 +95,15 @@ let private VSTestDefaults = TestAdapterPath = null } /// Builds the command line arguments from the given parameter record and the given assemblies. -let buildArgs (parameters : VSTestParams) (assembly: string) = +let buildArgs (parameters : VSTestParams) (assemblies: string seq) = let testsToRun = if not (Seq.isEmpty parameters.Tests) then sprintf @"/Tests:%s" (parameters.Tests |> String.separated ",") else null - new StringBuilder() - |> StringBuilder.appendIfTrue (assembly <> null) assembly + StringBuilder() + |> StringBuilder.appendFileNamesIfNotNull assemblies |> StringBuilder.appendIfNotNull parameters.SettingsPath "/Settings:" - |> StringBuilder.appendIfTrue (testsToRun <> null) testsToRun + |> StringBuilder.appendIfTrue (not (isNull testsToRun)) testsToRun |> StringBuilder.appendIfTrue parameters.Parallel "/Parallel" |> StringBuilder.appendIfTrue parameters.EnableCodeCoverage "/EnableCodeCoverage" |> StringBuilder.appendIfTrue parameters.InIsolation "/InIsolation" @@ -119,6 +120,31 @@ let buildArgs (parameters : VSTestParams) (assembly: string) = |> StringBuilder.appendIfNotNull parameters.TestAdapterPath "/TestAdapterPath:" |> StringBuilder.toText +let internal createProcess createTempFile (setParams : VSTestParams -> VSTestParams) (assemblies : string[]) = + let parameters = VSTestDefaults |> setParams + if Array.isEmpty assemblies then failwith "VSTest: cannot run tests (the assembly list is empty)." + let tool = parameters.ToolPath + let generatedArgs = buildArgs parameters assemblies + + let path = createTempFile() + let argLine = Args.toWindowsCommandLine [ (sprintf "@%s" path) ] + CreateProcess.fromRawCommandLine tool argLine + |> CreateProcess.withWorkingDirectory parameters.WorkingDir + |> CreateProcess.withTimeout parameters.TimeOut + |> CreateProcess.addOnSetup (fun () -> + File.WriteAllText(path, generatedArgs) + Trace.trace(sprintf "Saved args to '%s' with value: %s" path generatedArgs) + ) + |> CreateProcess.addOnFinally (fun () -> + File.Delete path + ) + |> CreateProcess.addOnExited (fun result exitCode -> + if exitCode > 0 && parameters.ErrorLevel <> ErrorLevel.DontFailBuild then + let message = sprintf "%sVSTest test run failed with exit code %i" Environment.NewLine exitCode + Trace.traceError message + failwith message + ) + /// Runs the VSTest command line tool (VSTest.Console.exe) on a group of assemblies. /// ## Parameters /// @@ -132,22 +158,10 @@ let buildArgs (parameters : VSTestParams) (assembly: string) = /// |> VSTest.run (fun p -> { p with SettingsPath = "Local.RunSettings" }) /// ) let run (setParams : VSTestParams -> VSTestParams) (assemblies : string seq) = - let details = assemblies |> String.separated ", " - use __ = Trace.traceTask "VSTest" details - let parameters = VSTestDefaults |> setParams - if String.IsNullOrEmpty parameters.ToolPath then failwith "VSTest: No tool path specified, or it could not be found automatically." let assemblies = assemblies |> Seq.toArray - if Array.isEmpty assemblies then failwith "VSTest: cannot run tests (the assembly list is empty)." - let failIfError assembly exitCode = - if exitCode > 0 && parameters.ErrorLevel <> ErrorLevel.DontFailBuild then - let message = sprintf "%sVSTest test run failed for %s" Environment.NewLine assembly - Trace.traceError message - failwith message - for assembly in assemblies do - let args = buildArgs parameters assembly - Process.execSimple (fun info -> - { info with - FileName = parameters.ToolPath - WorkingDirectory = parameters.WorkingDir - Arguments = args }) parameters.TimeOut - |> failIfError assembly + let details = assemblies |> String.separated ", " + use disposable = Trace.traceTask "VSTest" details + createProcess Path.GetTempFileName setParams assemblies + |> Proc.run + + disposable.MarkSuccess() From 1e7cb630c0380c0af6f649cf8436cec64c03774d Mon Sep 17 00:00:00 2001 From: Anh-Dung Phan Date: Mon, 26 Aug 2019 15:13:48 +0100 Subject: [PATCH 3/4] Add basic tests --- .../Fake.DotNet.Testing.VSTest.fsproj | 3 +- src/app/Fake.DotNet.Testing.VSTest/VSTest.fs | 2 +- .../Fake.DotNet.Testing.VSTest/VisibleTo.fs | 6 ++++ .../Fake.Core.UnitTests.fsproj | 2 ++ .../Fake.DotNet.Testing.VSTest.fs | 36 +++++++++++++++++++ 5 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 src/app/Fake.DotNet.Testing.VSTest/VisibleTo.fs create mode 100644 src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.VSTest.fs diff --git a/src/app/Fake.DotNet.Testing.VSTest/Fake.DotNet.Testing.VSTest.fsproj b/src/app/Fake.DotNet.Testing.VSTest/Fake.DotNet.Testing.VSTest.fsproj index 6ece046c028..a60a6de7e2b 100644 --- a/src/app/Fake.DotNet.Testing.VSTest/Fake.DotNet.Testing.VSTest.fsproj +++ b/src/app/Fake.DotNet.Testing.VSTest/Fake.DotNet.Testing.VSTest.fsproj @@ -1,4 +1,4 @@ - + netstandard2.0;net462 $(DefineConstants);NO_DOTNETCORE_BOOTSTRAP @@ -13,6 +13,7 @@ + diff --git a/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs b/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs index 426d99b50cb..dde72b97ba1 100644 --- a/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs +++ b/src/app/Fake.DotNet.Testing.VSTest/VSTest.fs @@ -86,7 +86,7 @@ let private VSTestDefaults = ListLoggers = false ListSettingsProviders = false ToolPath = - match Process.tryFindFile vsTestPaths vsTestExe with + match ProcessUtils.tryFindFile vsTestPaths vsTestExe with | Some path -> path | None -> "" WorkingDir = null diff --git a/src/app/Fake.DotNet.Testing.VSTest/VisibleTo.fs b/src/app/Fake.DotNet.Testing.VSTest/VisibleTo.fs new file mode 100644 index 00000000000..cdec1d13db0 --- /dev/null +++ b/src/app/Fake.DotNet.Testing.VSTest/VisibleTo.fs @@ -0,0 +1,6 @@ + +namespace System +open System.Runtime.CompilerServices + +[] +do () diff --git a/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj b/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj index 0d425b6be99..c1f9bfb2f4f 100644 --- a/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj +++ b/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj @@ -11,6 +11,7 @@ + @@ -41,6 +42,7 @@ + diff --git a/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.VSTest.fs b/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.VSTest.fs new file mode 100644 index 00000000000..9f4e2586318 --- /dev/null +++ b/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.VSTest.fs @@ -0,0 +1,36 @@ +module Fake.DotNet.Testing.VSTestTests + +open System.IO +open Fake.Core +open Fake.DotNet +open Fake.DotNet.Testing +open Fake.Testing +open Expecto + +[] +let tests = + testList "Fake.DotNet.Testing.VSTest.Tests" [ + testCase "Test that we write and delete arguments file" <| fun _ -> + let cp = + VSTest.createProcess Path.GetTempFileName (fun param -> + { param with + ToolPath = "vstest.exe" + }) [| "assembly.dll" |] + let file, args = + match cp.Command with + | RawCommand(file, args) -> file, args + | _ -> failwithf "expected RawCommand" + |> ArgumentHelper.checkIfMono + Expect.equal file "vstest.exe" "Expected vstest.exe" + Expect.equal (args |> Arguments.toArray).Length 1 "expected a single argument" + let arg = (args |> Arguments.toArray).[0] + Expect.stringStarts arg "@" "Expected arg to start with @" + let argFile = arg.Substring(1) + + ( use _state = cp.Hook.PrepareState() + let contents = File.ReadAllText argFile + let args = Args.fromWindowsCommandLine contents + Expect.sequenceEqual args ["assembly.dll"; "/InIsolation"] "Expected arg file to be correct" + ) + Expect.isFalse (File.Exists argFile) "File should be deleted" + ] From c700b06fd9423a61d69d33ae7272d54cd66f4372 Mon Sep 17 00:00:00 2001 From: Anh-Dung Phan Date: Mon, 26 Aug 2019 15:17:39 +0100 Subject: [PATCH 4/4] Add tests for the Parallel setting --- .../Fake.DotNet.Testing.VSTest.fs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.VSTest.fs b/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.VSTest.fs index 9f4e2586318..7df905a8e1d 100644 --- a/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.VSTest.fs +++ b/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.VSTest.fs @@ -33,4 +33,29 @@ let tests = Expect.sequenceEqual args ["assembly.dll"; "/InIsolation"] "Expected arg file to be correct" ) Expect.isFalse (File.Exists argFile) "File should be deleted" + + testCase "Test that we can set Parallel setting" <| fun _ -> + let cp = + VSTest.createProcess Path.GetTempFileName (fun param -> + { param with + ToolPath = "vstest.console.exe" + Parallel = true + }) [| "assembly1.dll"; "assembly2.dll" |] + let file, args = + match cp.Command with + | RawCommand(file, args) -> file, args + | _ -> failwithf "expected RawCommand" + |> ArgumentHelper.checkIfMono + Expect.equal file "vstest.console.exe" "Expected vstest.console.exe" + Expect.equal (args |> Arguments.toArray).Length 1 "expected a single argument" + let arg = (args |> Arguments.toArray).[0] + Expect.stringStarts arg "@" "Expected arg to start with @" + let argFile = arg.Substring(1) + + ( use _state = cp.Hook.PrepareState() + let contents = File.ReadAllText argFile + let args = Args.fromWindowsCommandLine contents + Expect.sequenceEqual args ["assembly1.dll"; "assembly2.dll"; "/Parallel"; "/InIsolation"] "Expected arg file to be correct" + ) + Expect.isFalse (File.Exists argFile) "File should be deleted" ]