diff --git a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs index e4366b4c8b..056c992f08 100644 --- a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs +++ b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs @@ -17,11 +17,13 @@ namespace Microsoft.VisualStudio.TestPlatform.Client.DesignMode using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers; + using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; + using Microsoft.VisualStudio.TestPlatform.Utilities; using CommunicationUtilitiesResources = Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Resources; using CoreUtilitiesConstants = Microsoft.VisualStudio.TestPlatform.CoreUtilities.Constants; using ObjectModelConstants = Microsoft.VisualStudio.TestPlatform.ObjectModel.Constants; @@ -199,6 +201,14 @@ private void ProcessRequests(ITestRequestManager testRequestManager) break; } + case MessageType.MultiTestRunsFinalization: + { + var multiTestRunsFinalizationPayload = + this.communicationManager.DeserializePayload(message); + this.FinalizeMultiTestRuns(multiTestRunsFinalizationPayload); + break; + } + case MessageType.CancelDiscovery: { testRequestManager.CancelDiscovery(); @@ -458,6 +468,41 @@ private void StartDiscovery(DiscoveryRequestPayload discoveryRequestPayload, ITe }); } + private void FinalizeMultiTestRuns(MultiTestRunsFinalizationPayload finalizationPayload) + { + lock(this.lockObject) + { + try + { + var handler = new MultiTestRunsDataCollectorAttachmentsHandler(new CodeCoverageDataAttachmentsHandler()); + handler.HandleAttachements(finalizationPayload.Attachments); + + var payload = new MultiTestRunsFinalizationCompletePayload() + { + Attachments = finalizationPayload.Attachments + }; + + // Send run complete to translation layer + this.communicationManager.SendMessage(MessageType.MultiTestRunsFinalizationComplete, payload); + } + catch (Exception ex) + { + EqtTrace.Error("DesignModeClient: Exception in StartDiscovery: " + ex); + + var testMessagePayload = new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = ex.ToString() }; + this.communicationManager.SendMessage(MessageType.TestMessage, testMessagePayload); + + var payload = new MultiTestRunsFinalizationCompletePayload() + { + Attachments = null + }; + + // Send run complete to translation layer + this.communicationManager.SendMessage(MessageType.MultiTestRunsFinalizationComplete, payload); + } + } + } + #region IDisposable Support private bool disposedValue = false; // To detect redundant calls diff --git a/src/testhost.x86/testhost.x86.csproj b/src/testhost.x86/testhost.x86.csproj index d35578cbfb..02dea17fb1 100644 --- a/src/testhost.x86/testhost.x86.csproj +++ b/src/testhost.x86/testhost.x86.csproj @@ -8,6 +8,7 @@ testhost.x86 netcoreapp2.1;net451 netcoreapp2.1 + 2.1.15 true AnyCPU true diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/MultiTestRunsFinalizationEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/MultiTestRunsFinalizationEventHandler.cs new file mode 100644 index 0000000000..b2bad52b2d --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/MultiTestRunsFinalizationEventHandler.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + + /// + public class MultiTestRunsFinalizationEventHandler : IMultiTestRunsFinalizationEventsHandler + { + /// + /// Gets the attachments. + /// + public List Attachments { get; private set; } + + /// + /// Gets the metrics. + /// + public IDictionary Metrics { get; private set; } + + /// + /// Gets the log message. + /// + public string LogMessage { get; private set; } + + public List Errors { get; set; } + + /// + /// Gets the test message level. + /// + public TestMessageLevel TestMessageLevel { get; private set; } + + public MultiTestRunsFinalizationEventHandler() + { + this.Errors = new List(); + this.Attachments = new List(); + } + + public void EnsureSuccess() + { + if (this.Errors.Any()) + { + throw new InvalidOperationException($"Test run reported errors:{Environment.NewLine}{string.Join(Environment.NewLine + Environment.NewLine, this.Errors)}"); + } + } + + public void HandleLogMessage(TestMessageLevel level, string message) + { + this.LogMessage = message; + this.TestMessageLevel = level; + if (level == TestMessageLevel.Error) { + this.Errors.Add(message); + } + } + + public void HandleRawMessage(string rawMessage) + { + // No op + } + + public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessStartInfo) + { + // No op + return -1; + } + + public bool AttachDebuggerToProcess(int pid) + { + // No op + return true; + } + + public void HandleMultiTestRunsFinalizationComplete(ICollection attachments) + { + if(attachments != null) + { + this.Attachments.AddRange(attachments); + } + } + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs index f3e1793d30..6b694e29a9 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs @@ -18,6 +18,11 @@ public class RunEventHandler : ITestRunEventsHandler2 /// public List TestResults { get; private set; } + /// + /// Gets the attachments. + /// + public List Attachments { get; private set; } + /// /// Gets the metrics. /// @@ -39,6 +44,7 @@ public RunEventHandler() { this.TestResults = new List(); this.Errors = new List(); + this.Attachments = new List(); } public void EnsureSuccess() @@ -69,6 +75,11 @@ public void HandleTestRunComplete( this.TestResults.AddRange(lastChunkArgs.NewTestResults); } + if(runContextAttachments != null) + { + this.Attachments.AddRange(runContextAttachments); + } + this.Metrics = testRunCompleteArgs.Metrics; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/FinalizeMultiTestRunsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/FinalizeMultiTestRunsTests.cs new file mode 100644 index 0000000000..433eca8a26 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/FinalizeMultiTestRunsTests.cs @@ -0,0 +1,243 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests +{ + using Microsoft.TestPlatform.TestUtilities; + using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; + using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Linq; + using VisualStudio.TestPlatform.ObjectModel.Logging; + + /// + /// The Run Tests using VsTestConsoleWrapper API's + /// + [TestClass] + public class FinalizeMultiTestRunsTests : AcceptanceTestBase + { + private IVsTestConsoleWrapper2 vstestConsoleWrapper; + private RunEventHandler runEventHandler; + private MultiTestRunsFinalizationEventHandler multiTestRunsFinalizationEventHandler; + + private void Setup() + { + this.vstestConsoleWrapper = this.GetVsTestConsoleWrapper(); + this.runEventHandler = new RunEventHandler(); + this.multiTestRunsFinalizationEventHandler = new MultiTestRunsFinalizationEventHandler(); + } + + [TestCleanup] + public void Cleanup() + { + this.vstestConsoleWrapper?.EndSession(); + } + + [TestMethod] + //[NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] + public void RunAllTests(RunnerInfo runnerInfo) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); + this.Setup(); + + this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Take(1), this.GetCodeCoverageRunSettings(), this.runEventHandler); + this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Skip(1), this.GetCodeCoverageRunSettings(), this.runEventHandler); + + // Assert + Assert.AreEqual(6, this.runEventHandler.TestResults.Count); + Assert.AreEqual(2, this.runEventHandler.Attachments.Count); + Assert.AreEqual(2, this.runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); + Assert.AreEqual(2, this.runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); + Assert.AreEqual(2, this.runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); + + this.vstestConsoleWrapper.FinalizeMultiTestRuns(runEventHandler.Attachments, multiTestRunsFinalizationEventHandler); + Assert.AreEqual(1, this.multiTestRunsFinalizationEventHandler.Attachments.Count); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] + public void EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runnerInfo) + { + var numOfProcesses = Process.GetProcessesByName("vstest.console").Length; + + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); + this.Setup(); + + this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies(), this.GetDefaultRunSettings(), this.runEventHandler); + this.vstestConsoleWrapper?.EndSession(); + + // Assert + Assert.AreEqual(numOfProcesses, Process.GetProcessesByName("vstest.console").Length); + + this.vstestConsoleWrapper = null; + } + + [TestMethod] + [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] + public void RunTestsWithTelemetryOptedIn(RunnerInfo runnerInfo) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); + this.Setup(); + + this.vstestConsoleWrapper.RunTests( + this.GetTestAssemblies(), + this.GetDefaultRunSettings(), + new TestPlatformOptions() { CollectMetrics = true }, + this.runEventHandler); + + // Assert + Assert.AreEqual(6, this.runEventHandler.TestResults.Count); + Assert.IsTrue(this.runEventHandler.Metrics.ContainsKey(TelemetryDataConstants.TargetDevice)); + Assert.IsTrue(this.runEventHandler.Metrics.ContainsKey(TelemetryDataConstants.TargetFramework)); + Assert.IsTrue(this.runEventHandler.Metrics.ContainsKey(TelemetryDataConstants.TargetOS)); + Assert.IsTrue(this.runEventHandler.Metrics.ContainsKey(TelemetryDataConstants.TimeTakenInSecForRun)); + Assert.IsTrue(this.runEventHandler.Metrics.ContainsKey(TelemetryDataConstants.NumberOfAdapterDiscoveredDuringExecution)); + Assert.IsTrue(this.runEventHandler.Metrics.ContainsKey(TelemetryDataConstants.RunState)); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] + public void RunTestsWithTelemetryOptedOut(RunnerInfo runnerInfo) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); + this.Setup(); + + this.vstestConsoleWrapper.RunTests( + this.GetTestAssemblies(), + this.GetDefaultRunSettings(), + new TestPlatformOptions() { CollectMetrics = false }, + this.runEventHandler); + + // Assert + Assert.AreEqual(6, this.runEventHandler.TestResults.Count); + Assert.AreEqual(0, this.runEventHandler.Metrics.Count); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] + public void RunTestsShouldThrowOnStackOverflowException(RunnerInfo runnerInfo) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); + this.Setup(); + + if (IntegrationTestEnvironment.BuildConfiguration.Equals("release", StringComparison.OrdinalIgnoreCase)) + { + // On release, x64 builds, recursive calls may be replaced with loops (tail call optimization) + Assert.Inconclusive("On StackOverflowException testhost not exited in release configuration."); + return; + } + + var source = new[] { this.GetAssetFullPath("SimpleTestProject3.dll") }; + + this.vstestConsoleWrapper.RunTests( + source, + this.GetDefaultRunSettings(), + new TestPlatformOptions() { TestCaseFilter = "ExitWithStackoverFlow" }, + this.runEventHandler); + + var errorMessage = runnerInfo.TargetFramework == "net451" + ? "The active test run was aborted. Reason: Test host process crashed : Process is terminated due to StackOverflowException.\r\n" + : "The active test run was aborted. Reason: Test host process crashed : Process is terminating due to StackOverflowException.\r\n"; + + Assert.AreEqual(errorMessage, this.runEventHandler.LogMessage); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource(useCoreRunner: false)] + [NetCoreTargetFrameworkDataSource(useCoreRunner: false)] + public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo runnerInfo) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); + this.Setup(); + + var testAssemblyName = "SimpleTestProject2.dll"; + var source = new List() { this.GetAssetFullPath(testAssemblyName) }; + + var veryLongTestCaseFilter = + "FullyQualifiedName=VeryLongTestCaseNameeeeeeeeeeeeee" + + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; + + this.vstestConsoleWrapper.RunTests( + source, + this.GetDefaultRunSettings(), + new TestPlatformOptions() { TestCaseFilter = veryLongTestCaseFilter }, + this.runEventHandler); + + var expectedFilter = veryLongTestCaseFilter.Substring(0, 256) + "..."; + + // Assert + StringAssert.StartsWith(this.runEventHandler.LogMessage, $"No test matches the given testcase filter `{expectedFilter}` in"); + StringAssert.EndsWith(this.runEventHandler.LogMessage, testAssemblyName); + + Assert.AreEqual(TestMessageLevel.Warning, this.runEventHandler.TestMessageLevel); + } + + private IList GetTestAssemblies() + { + var testAssemblies = new List + { + this.GetAssetFullPath("SimpleTestProject.dll"), + this.GetAssetFullPath("SimpleTestProject2.dll") + }; + + return testAssemblies; + } + + /// + /// Default RunSettings + /// + /// + public string GetCodeCoverageRunSettings() + { + string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, $@"src\DataCollectors\TraceDataCollector\bin\{IntegrationTestEnvironment.BuildConfiguration}\netstandard2.0"); + if (this.testEnvironment.TargetFramework.Equals(IntegrationTestBase.DesktopRunnerFramework)) + { + traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, $@"artifacts\{IntegrationTestEnvironment.BuildConfiguration}\Microsoft.CodeCoverage"); + } + + string runSettingsXml = $@" + + + {FrameworkArgValue} + {traceDataCollectorDir} + + + + + + + + + .*CPPUnitTestFramework.* + + + + + True + True + True + False + + + + + + "; + return runSettingsXml; + } + } +} \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index 9be9c83360..87ed476082 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -21,7 +21,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests [TestClass] public class RunTests : AcceptanceTestBase { - private IVsTestConsoleWrapper vstestConsoleWrapper; + private IVsTestConsoleWrapper2 vstestConsoleWrapper; private RunEventHandler runEventHandler; private void Setup() diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 7d25744753..44356520c2 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -451,7 +451,7 @@ protected virtual string SetVSTestConsoleDLLPathInArgs(string args) /// Returns the VsTestConsole Wrapper. /// /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper() + public IVsTestConsoleWrapper2 GetVsTestConsoleWrapper() { var logFileName = Path.GetFileName(Path.GetTempFileName()); var logFileDir = Path.Combine(Path.GetTempPath(), "VSTestConsoleWrapperLogs");