diff --git a/TestFx.sln b/TestFx.sln index 956ed1d772..b3650f547f 100644 --- a/TestFx.sln +++ b/TestFx.sln @@ -178,7 +178,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CompatTestProject", "test\E EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestProjectForAssemblyResolution", "test\ComponentTests\TestAssets\TestProjectForAssemblyResolution\TestProjectForAssemblyResolution.csproj", "{0B057B99-DCDD-417A-BC19-3E63DDD86F24}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataRowTestProject", "test\E2ETests\TestAssets\DataRowTestProject\DataRowTestProject.csproj", "{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataRowTestProject", "test\E2ETests\TestAssets\DataRowTestProject\DataRowTestProject.csproj", "{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TimeoutTestProject", "test\E2ETests\TestAssets\TimeoutTestProject\TimeoutTestProject.csproj", "{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}" EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution @@ -1065,6 +1067,30 @@ Global {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x64.Build.0 = Release|Any CPU {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x86.ActiveCfg = Release|Any CPU {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x86.Build.0 = Release|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|ARM.Build.0 = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|x64.ActiveCfg = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|x64.Build.0 = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|x86.ActiveCfg = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|x86.Build.0 = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|ARM.ActiveCfg = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|ARM.Build.0 = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|x64.ActiveCfg = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|x64.Build.0 = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|x86.ActiveCfg = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|x86.Build.0 = Debug|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|Any CPU.Build.0 = Release|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|ARM.ActiveCfg = Release|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|ARM.Build.0 = Release|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|x64.ActiveCfg = Release|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|x64.Build.0 = Release|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|x86.ActiveCfg = Release|Any CPU + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1125,6 +1151,7 @@ Global {2D2C5B73-F1F1-47C8-BC5C-A172E9BB3D16} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} {0B057B99-DCDD-417A-BC19-3E63DDD86F24} = {1899187D-8B9C-40C2-9F04-9E9A76C9A919} {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} + {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9} diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs index 8a36f0c174..059adf2d0a 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs @@ -10,6 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution using System.Globalization; using System.Reflection; using System.Text; + using System.Threading; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; @@ -453,8 +454,8 @@ private void RunTestCleanupMethod(object classInstance, TestResult result) if (cleanupStackTrace.Length > 0) { - cleanupStackTrace.Append(Resource.UTA_CleanupStackTrace); - cleanupStackTrace.Append(Environment.NewLine); + cleanupStackTrace.Append(Resource.UTA_CleanupStackTrace); + cleanupStackTrace.Append(Environment.NewLine); } Exception realException = ex.GetInnerExceptionOrDefault(); @@ -668,40 +669,42 @@ private TestResult ExecuteInternalWithTimeout(object[] arguments) TestResult result = null; Exception failure = null; - Action executeAsyncAction = () => + void executeAsyncAction() + { + try { - try - { - result = this.ExecuteInternal(arguments); - } - catch (Exception ex) - { - failure = ex; - } - }; + result = this.ExecuteInternal(arguments); + } + catch (Exception ex) + { + failure = ex; + } + } - if (PlatformServiceProvider.Instance.ThreadOperations.Execute(executeAsyncAction, this.TestMethodOptions.Timeout)) + CancellationToken cancelToken = this.TestMethodOptions.TestContext.Context.CancellationTokenSource.Token; + if (PlatformServiceProvider.Instance.ThreadOperations.Execute(executeAsyncAction, this.TestMethodOptions.Timeout, cancelToken)) { if (failure != null) { throw failure; } - Debug.Assert(result != null, "no timeout, no failure result should not be null"); return result; } else { - // Timed out - - // If the method times out, then - // - // 1. If the test is stuck, then we can get CannotUnloadAppDomain exception. - // - // Which are handled as follows: - - // - // For #1, we are now restarting the execution process if adapter fails to unload app-domain. + // Timed out or canceled string errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, this.TestMethodName); + if (this.TestMethodOptions.TestContext.Context.CancellationTokenSource.IsCancellationRequested) + { + errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Cancelled, this.TestMethodName); + } + else + { + // Cancel the token source as test has timedout + this.TestMethodOptions.TestContext.Context.CancellationTokenSource.Cancel(); + } + TestResult timeoutResult = new TestResult() { Outcome = TestTools.UnitTesting.UnitTestOutcome.Timeout, TestFailureException = new TestFailedException(UnitTestOutcome.Timeout, errorMessage) }; return timeoutResult; } diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index fc7b8b57ed..cad0203f89 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -183,13 +183,14 @@ internal UnitTestResult[] Execute() result = new[] { new UnitTestResult() }; } - var newResult = - new UnitTestResult(new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation())); - newResult.StandardOut = result[result.Length - 1].StandardOut; - newResult.StandardError = result[result.Length - 1].StandardError; - newResult.DebugTrace = result[result.Length - 1].DebugTrace; - newResult.TestContextMessages = result[result.Length - 1].TestContextMessages; - newResult.Duration = result[result.Length - 1].Duration; + var newResult = new UnitTestResult(new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation())) + { + StandardOut = result[result.Length - 1].StandardOut, + StandardError = result[result.Length - 1].StandardError, + DebugTrace = result[result.Length - 1].DebugTrace, + TestContextMessages = result[result.Length - 1].TestContextMessages, + Duration = result[result.Length - 1].Duration + }; result[result.Length - 1] = newResult; } finally diff --git a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs index 155b9f0748..c57f4ac4a3 100644 --- a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs +++ b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs @@ -142,6 +142,15 @@ internal static string EnumeratorLoadTypeErrorFormat { } } + /// + /// Looks up a localized string similar to Test '{0}' execution has been aborted.. + /// + internal static string Execution_Test_Cancelled { + get { + return ResourceManager.GetString("Execution_Test_Cancelled", resourceCulture); + } + } + /// /// Looks up a localized string similar to Test '{0}' exceeded execution timeout period.. /// diff --git a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx index 5ab63fe73f..2b576f22e6 100644 --- a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx +++ b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx @@ -317,4 +317,7 @@ Error: {1} Only data driven test methods can have parameters. Did you intend to use [DataRow] or [DynamicData]? + + Test '{0}' execution has been aborted. + \ No newline at end of file diff --git a/src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs b/src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs index b625eebbfc..a7373ba5b4 100644 --- a/src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs +++ b/src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs @@ -13,6 +13,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices using System.Globalization; using System.IO; using System.Linq; + using System.Threading; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Deployment; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.ObjectModel; @@ -82,7 +83,7 @@ public TestContextImplementation(ITestMethod testMethod, StringWriter stringWrit this.testMethod = testMethod; this.stringWriter = stringWriter; this.properties = new Dictionary(properties); - + this.CancellationTokenSource = new CancellationTokenSource(); this.InitializeProperties(); this.testResultFiles = new List(); diff --git a/src/Adapter/PlatformServices.Desktop/Services/DesktopThreadOperations.cs b/src/Adapter/PlatformServices.Desktop/Services/DesktopThreadOperations.cs index c670ec1740..4d8cfbb595 100644 --- a/src/Adapter/PlatformServices.Desktop/Services/DesktopThreadOperations.cs +++ b/src/Adapter/PlatformServices.Desktop/Services/DesktopThreadOperations.cs @@ -8,6 +8,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices using System.Threading; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; #pragma warning disable SA1649 // SA1649FileNameMustMatchTypeName @@ -21,36 +22,44 @@ public class ThreadOperations : IThreadOperations /// /// The action to execute. /// Timeout for the specified action in milliseconds. + /// Token to cancel the execution /// Returns true if the action executed before the timeout. returns false otherwise. - public bool Execute(Action action, int timeout) + public bool Execute(Action action, int timeout, CancellationToken cancelToken) { - Thread executionThread = new Thread(new ThreadStart(action)); - executionThread.IsBackground = true; - executionThread.Name = "MSTestAdapter Thread"; + bool executionAborted = false; + Thread executionThread = new Thread(new ThreadStart(action)) + { + IsBackground = true, + Name = "MSTestAdapter Thread" + }; executionThread.SetApartmentState(Thread.CurrentThread.GetApartmentState()); executionThread.Start(); + cancelToken.Register(() => + { + executionAborted = true; + AbortThread(executionThread); + }); - if (executionThread.Join(timeout)) + if (JoinThread(timeout, executionThread)) { + if (executionAborted) + { + return false; + } + + // Successfully completed return true; } + else if (executionAborted) + { + // Execution aborted due to user choice + return false; + } else { // Timed out - try - { - // Abort test thread after timeout. - executionThread.Abort(); - } - catch (ThreadStateException) - { - // Catch and discard ThreadStateException. If Abort is called on a thread that has been suspended, - // a ThreadStateException is thrown in the thread that called Abort, - // and AbortRequested is added to the ThreadState property of the thread being aborted. - // A ThreadAbortException is not thrown in the suspended thread until Resume is called. - } - + AbortThread(executionThread); return false; } } @@ -75,7 +84,36 @@ public void ExecuteWithAbortSafety(Action action) throw new TargetInvocationException(exception); } } - } + private static bool JoinThread(int timeout, Thread executionThread) + { + try + { + return executionThread.Join(timeout); + } + catch (ThreadStateException) + { + // Join was called on a thread not started + } + + return false; + } + + private static void AbortThread(Thread executionThread) + { + try + { + // Abort test thread after timeout. + executionThread.Abort(); + } + catch (ThreadStateException) + { + // Catch and discard ThreadStateException. If Abort is called on a thread that has been suspended, + // a ThreadStateException is thrown in the thread that called Abort, + // and AbortRequested is added to the ThreadState property of the thread being aborted. + // A ThreadAbortException is not thrown in the suspended thread until Resume is called. + } + } + } #pragma warning restore SA1649 // SA1649FileNameMustMatchTypeName } diff --git a/src/Adapter/PlatformServices.Interface/IThreadOperations.cs b/src/Adapter/PlatformServices.Interface/IThreadOperations.cs index a2dcf15ac7..c914b09a95 100644 --- a/src/Adapter/PlatformServices.Interface/IThreadOperations.cs +++ b/src/Adapter/PlatformServices.Interface/IThreadOperations.cs @@ -4,6 +4,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface { using System; + using System.Threading; /// /// This service is responsible for any thread operations specific to a platform. @@ -15,8 +16,9 @@ public interface IThreadOperations /// /// The action to execute. /// Timeout for the specified action. + /// Token to cancel the execution /// Returns true if the action executed before the timeout. returns false otherwise. - bool Execute(Action action, int timeout); + bool Execute(Action action, int timeout, CancellationToken cancelToken); /// /// Execute an action with handling for Thread Aborts (if possible) so the main thread of the adapter does not die. diff --git a/src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10TestContextImplementation.cs b/src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10TestContextImplementation.cs index 48d1ffd431..b0b090992c 100644 --- a/src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10TestContextImplementation.cs +++ b/src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10TestContextImplementation.cs @@ -9,7 +9,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices using System.Diagnostics; using System.Globalization; using System.IO; - + using System.Threading; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.ObjectModel; @@ -61,6 +61,7 @@ public TestContextImplementation(ITestMethod testMethod, StringWriter writer, ID this.testMethod = testMethod; this.properties = new Dictionary(properties); this.stringWriter = writer; + this.CancellationTokenSource = new CancellationTokenSource(); this.InitializeProperties(); } diff --git a/src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10ThreadOperations.cs b/src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10ThreadOperations.cs index 5c0228d8ac..cb2f8f18b3 100644 --- a/src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10ThreadOperations.cs +++ b/src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10ThreadOperations.cs @@ -4,6 +4,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices { using System; + using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; @@ -20,12 +21,12 @@ public class ThreadOperations : IThreadOperations /// /// The action to execute. /// Timeout for the specified action. + /// Token to cancel the execution /// Returns true if the action executed before the timeout. returns false otherwise. - public bool Execute(Action action, int timeout) + public bool Execute(Action action, int timeout, CancellationToken cancelToken) { var executionTask = Task.Factory.StartNew(action); - - if (executionTask.Wait(timeout)) + if (executionTask.Wait(timeout, cancelToken)) { return true; } diff --git a/src/TestFramework/Extension.Desktop/TestContext.cs b/src/TestFramework/Extension.Desktop/TestContext.cs index a6f019bbcc..8ef0694658 100644 --- a/src/TestFramework/Extension.Desktop/TestContext.cs +++ b/src/TestFramework/Extension.Desktop/TestContext.cs @@ -11,6 +11,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; + using System.Threading; /// /// Used to store information that is provided to unit tests. @@ -22,6 +23,11 @@ public abstract class TestContext /// public abstract IDictionary Properties { get; } + /// + /// Gets or sets the cancellation token source. This token source is cancelled when test timesout. Also when explicitly cancelled the test will be aborted + /// + public virtual CancellationTokenSource CancellationTokenSource { get; protected set; } + /// /// Gets the current data row when test is used for data driven testing. /// diff --git a/src/TestFramework/Extension.Shared/TestContext.cs b/src/TestFramework/Extension.Shared/TestContext.cs index 225a7f3c18..73996020eb 100644 --- a/src/TestFramework/Extension.Shared/TestContext.cs +++ b/src/TestFramework/Extension.Shared/TestContext.cs @@ -8,6 +8,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting using System.Collections.Generic; using System.Diagnostics; using System.Globalization; + using System.Threading; /// /// TestContext class. This class should be fully abstract and not contain any @@ -21,6 +22,11 @@ public abstract class TestContext /// public abstract IDictionary Properties { get; } + /// + /// Gets or sets the cancellation token source. This token source is cancelled when test timesout. Also when explicitly cancelled the test will be aborted + /// + public virtual CancellationTokenSource CancellationTokenSource { get; protected set; } + /// /// Gets Fully-qualified name of the class containing the test method currently being executed /// diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index 42b9cad4a9..61b6062a76 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -139,7 +139,7 @@ public void ValidatePassedTestsCount(int expectedPassedTestsCount) public void ValidateFailedTests(string source, params string[] failedTests) { this.ValidateFailedTestsCount(failedTests.Length); - this.ValidateFailedTestsContain(source, failedTests); + this.ValidateFailedTestsContain(source, true, failedTests); } /// @@ -185,12 +185,13 @@ public void ValidatePassedTestsContain(params string[] passedTests) /// Validates if the test results contains the specified set of failed tests. /// /// The test container. + /// Validates the existence of stack trace when set to true /// Set of failed tests. /// /// Provide the full test name similar to this format SampleTest.TestCode.TestMethodFailed. /// Also validates whether these tests have stack trace info. /// - public void ValidateFailedTestsContain(string source, params string[] failedTests) + public void ValidateFailedTestsContain(string source, bool validateStackTraceInfo, params string[] failedTests) { foreach (var test in failedTests) { @@ -199,7 +200,7 @@ public void ValidateFailedTestsContain(string source, params string[] failedTest Assert.IsNotNull(testFound, "Test {0} does not appear in failed tests list.", test); // Skipping this check for x64 as of now. https://github.com/Microsoft/testfx/issues/60 should fix this. - if (source.IndexOf("x64") == -1) + if (source.IndexOf("x64") == -1 && validateStackTraceInfo) { // Verify stack information as well. Assert.IsTrue(testFound.ErrorStackTrace.Contains(GetTestMethodName(test)), "No stack trace for failed test: {0}", test); diff --git a/test/E2ETests/Smoke.E2E.Tests/AssertExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/AssertExtensibilityTests.cs index 480ca13f1a..e938258afd 100644 --- a/test/E2ETests/Smoke.E2E.Tests/AssertExtensibilityTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/AssertExtensibilityTests.cs @@ -21,6 +21,7 @@ public void ExecuteAssertExtensibilityTests() this.ValidateFailedTestsContain( TestAssembly, + true, "FxExtensibilityTestProject.AssertExTest.BasicFailingAssertExtensionTest", "FxExtensibilityTestProject.AssertExTest.ChainedFailingAssertExtensionTest"); } diff --git a/test/E2ETests/Smoke.E2E.Tests/CompatTests.cs b/test/E2ETests/Smoke.E2E.Tests/CompatTests.cs index 9c6bebbaf0..b8e7dfad95 100644 --- a/test/E2ETests/Smoke.E2E.Tests/CompatTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/CompatTests.cs @@ -38,7 +38,13 @@ public void RunAllCompatTests() "SampleUnitTestProject.UnitTest1.PassingTest"); this.ValidateFailedTestsContain( - "CompatTestProject.UnitTest1.FailingTest", + OldAdapterTestProject, + true, + "CompatTestProject.UnitTest1.FailingTest"); + + this.ValidateFailedTestsContain( + LatestAdapterTestProject, + true, "SampleUnitTestProject.UnitTest1.FailingTest"); this.ValidateSkippedTestsContain( diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs index eea8e4b3d1..0358dc9835 100644 --- a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs @@ -26,6 +26,7 @@ public void ExecuteCustomTestExtensibilityTests() "CustomTestClass1 - Execution number 5"); this.ValidateFailedTestsContain( TestAssembly, + true, "CustomTestMethod1 - Execution number 3", "CustomTestClass1 - Execution number 3"); } @@ -43,6 +44,7 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests() this.ValidateFailedTestsCount(7); this.ValidateFailedTestsContain( TestAssembly, + true, "CustomTestMethod2 (A)", "CustomTestMethod2 (A)", "CustomTestMethod2 (A)", diff --git a/test/E2ETests/Smoke.E2E.Tests/DataSourceTests.cs b/test/E2ETests/Smoke.E2E.Tests/DataSourceTests.cs index 2224f2f6f2..bf23352a0e 100644 --- a/test/E2ETests/Smoke.E2E.Tests/DataSourceTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/DataSourceTests.cs @@ -21,6 +21,8 @@ public void ExecuteCsvTestDataSourceTests() "CsvTestMethod (Data Row 2)"); this.ValidateFailedTestsContain( + TestAssembly, + true, "CsvTestMethod (Data Row 1)", "CsvTestMethod (Data Row 3)"); } diff --git a/test/E2ETests/Smoke.E2E.Tests/DeploymentTests.cs b/test/E2ETests/Smoke.E2E.Tests/DeploymentTests.cs index d1270f6c34..645363c0e1 100644 --- a/test/E2ETests/Smoke.E2E.Tests/DeploymentTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/DeploymentTests.cs @@ -22,7 +22,7 @@ public void ValidateTestSourceDependencyDeployment() { this.InvokeVsTestForExecution(new string[] { TestAssembly }); this.ValidatePassedTestsContain("DeploymentTestProject.UnitTest1.FailIfFilePresent", "DeploymentTestProject.UnitTest1.PassIfDeclaredFilesPresent"); - this.ValidateFailedTestsContain("DeploymentTestProject.dll", "DeploymentTestProject.UnitTest1.PassIfFilePresent"); + this.ValidateFailedTestsContain("DeploymentTestProject.dll", true, "DeploymentTestProject.UnitTest1.PassIfFilePresent"); } [TestMethod] @@ -30,7 +30,7 @@ public void ValidateTestSourceLocationDeployment() { this.InvokeVsTestForExecution(new string[] { TestAssembly }, RunSetting); this.ValidatePassedTestsContain("DeploymentTestProject.UnitTest1.PassIfFilePresent", "DeploymentTestProject.UnitTest1.PassIfDeclaredFilesPresent"); - this.ValidateFailedTestsContain("DeploymentTestProject.dll", "DeploymentTestProject.UnitTest1.FailIfFilePresent"); + this.ValidateFailedTestsContain("DeploymentTestProject.dll", true, "DeploymentTestProject.UnitTest1.FailIfFilePresent"); } } } diff --git a/test/E2ETests/Smoke.E2E.Tests/ParallelExecutionTests.cs b/test/E2ETests/Smoke.E2E.Tests/ParallelExecutionTests.cs index b2b402a86f..3c3a03af23 100644 --- a/test/E2ETests/Smoke.E2E.Tests/ParallelExecutionTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/ParallelExecutionTests.cs @@ -86,6 +86,8 @@ public void NothingShouldRunInParallel() "DoNotParallelizeTestProject.UnitTest2.SimpleTest21"); this.ValidateFailedTestsContain( + DoNotParallelizeTestAssembly, + true, "DoNotParallelizeTestProject.UnitTest1.SimpleTest12", "DoNotParallelizeTestProject.UnitTest2.SimpleTest22"); } diff --git a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj index ffc75ae641..cf9db9e8d6 100644 --- a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj +++ b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj @@ -39,6 +39,7 @@ + diff --git a/test/E2ETests/Smoke.E2E.Tests/TimeoutTests.cs b/test/E2ETests/Smoke.E2E.Tests/TimeoutTests.cs new file mode 100644 index 0000000000..f11e87c08d --- /dev/null +++ b/test/E2ETests/Smoke.E2E.Tests/TimeoutTests.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace MSTestAdapter.Smoke.E2ETests +{ + using System.IO; + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class TimeoutTests : CLITestBase + { + private const string TimeoutTestAssembly = "TimeoutTestProject.dll"; + private const int TestMethodWaitTimeInMs = 6000; + private const int OverheadTimeInMs = 2500; + + [TestMethod] + public void ValidateTimeoutTests() + { + this.InvokeVsTestForExecution(new string[] { TimeoutTestAssembly }); + + this.ValidateTestRunTime(TestMethodWaitTimeInMs + OverheadTimeInMs); + + this.ValidateFailedTestsCount(2); + + this.ValidateFailedTestsContain( + TimeoutTestAssembly, + false, + "TimeoutTestProject.TerimnateLongRunningTasksUsingTokenTestClass.TerimnateLongRunningTasksUsingToken", + "TimeoutTestProject.SelfTerminatingTestClass.SelfTerminatingTestMethod"); + + Assert.IsTrue(File.Exists(this.GetAssetFullPath("TimeoutTestOutput.txt")), "Unable to locate the TimeoutTestOutput.txt file"); + } + } +} diff --git a/test/E2ETests/TestAssets/TimeoutTestProject/SelfTerminatingTestClass.cs b/test/E2ETests/TestAssets/TimeoutTestProject/SelfTerminatingTestClass.cs new file mode 100644 index 0000000000..9945962493 --- /dev/null +++ b/test/E2ETests/TestAssets/TimeoutTestProject/SelfTerminatingTestClass.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace TimeoutTestProject +{ + [TestClass] + public class SelfTerminatingTestClass + { + public TestContext TestContext { get; set; } + + [TestMethod] + [Timeout(60000)] + public void SelfTerminatingTestMethod() + { + TestContext.CancellationTokenSource.Cancel(); + } + } +} diff --git a/test/E2ETests/TestAssets/TimeoutTestProject/TerimnateLongRunningTasksUsingTokenTestClass.cs b/test/E2ETests/TestAssets/TimeoutTestProject/TerimnateLongRunningTasksUsingTokenTestClass.cs new file mode 100644 index 0000000000..808f7af935 --- /dev/null +++ b/test/E2ETests/TestAssets/TimeoutTestProject/TerimnateLongRunningTasksUsingTokenTestClass.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace TimeoutTestProject +{ + [TestClass] + public class TerimnateLongRunningTasksUsingTokenTestClass + { + public TestContext TestContext { get; set; } + + [TestMethod] + [Timeout(5000)] + public void TerimnateLongRunningTasksUsingToken() + { + var longTask = new Thread(ExecuteLong); + longTask.Start(); + longTask.Join(); + } + + private void ExecuteLong() + { + try + { + File.Delete("TimeoutTestOutput.txt"); + Task.Delay(100000).Wait(TestContext.CancellationTokenSource.Token); + } + catch (OperationCanceledException) + { + File.WriteAllText("TimeoutTestOutput.txt", "Written from long running thread post termination"); + } + } + } +} diff --git a/test/E2ETests/TestAssets/TimeoutTestProject/TimeoutTestProject.csproj b/test/E2ETests/TestAssets/TimeoutTestProject/TimeoutTestProject.csproj new file mode 100644 index 0000000000..482fd0b45e --- /dev/null +++ b/test/E2ETests/TestAssets/TimeoutTestProject/TimeoutTestProject.csproj @@ -0,0 +1,17 @@ + + + ..\..\..\..\ + + + + net452 + false + false + $(TestFxRoot)artifacts\TestAssets\ + + + + + + + diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodInfoTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodInfoTests.cs index d450302f6f..2d6408bc16 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodInfoTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodInfoTests.cs @@ -14,6 +14,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution using System.Linq; using System.Reflection; using System.Text; + using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; @@ -371,6 +372,9 @@ public void TestMethodInfoInvokeShouldSetResultFilesIfTestContextHasAttachments( { Mock testContext = new Mock(); testContext.Setup(tc => tc.GetResultFiles()).Returns(new List() { "C:\\temp.txt" }); + var mockInnerContext = new Mock(); + testContext.SetupGet(tc => tc.Context).Returns(mockInnerContext.Object); + mockInnerContext.SetupGet(tc => tc.CancellationTokenSource).Returns(new CancellationTokenSource()); this.testMethodOptions.TestContext = testContext.Object; var method = new TestMethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions); @@ -1221,7 +1225,7 @@ public void TestMethodInfoInvokeShouldReturnTestFailureOnTimeout() PlatformServiceProvider.Instance = testablePlatformServiceProvider; testablePlatformServiceProvider.MockThreadOperations.Setup( - to => to.Execute(It.IsAny(), It.IsAny())).Returns(false); + to => to.Execute(It.IsAny(), It.IsAny(), It.IsAny())).Returns(false); this.testMethodOptions.Timeout = 1; var method = new TestMethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions); @@ -1235,18 +1239,66 @@ public void TestMethodInfoInvokeShouldReturnTestFailureOnTimeout() [TestMethodV1] public void TestMethodInfoInvokeShouldReturnTestPassedOnCompletionWithinTimeout() { - DummyTestClass.TestMethodBody = o => - { - /* do nothing */ - }; - + DummyTestClass.TestMethodBody = o => { /* do nothing */ }; var method = new TestMethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions); - var result = method.Invoke(null); - Assert.AreEqual(UTF.UnitTestOutcome.Passed, result.Outcome); } + [TestMethodV1] + public void TestMethodInfoInvokeShouldCancelTokenSourceOnTimeout() + { + var testablePlatformServiceProvider = new TestablePlatformServiceProvider(); + this.RunWithTestablePlatformService(testablePlatformServiceProvider, () => + { + testablePlatformServiceProvider.MockThreadOperations.CallBase = true; + PlatformServiceProvider.Instance = testablePlatformServiceProvider; + + testablePlatformServiceProvider.MockThreadOperations.Setup( + to => to.Execute(It.IsAny(), It.IsAny(), It.IsAny())).Returns(false); + this.testMethodOptions.Timeout = 1; + + var method = new TestMethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions); + var result = method.Invoke(null); + + Assert.AreEqual(UTF.UnitTestOutcome.Timeout, result.Outcome); + StringAssert.Contains(result.TestFailureException.Message, "exceeded execution timeout period"); + Assert.IsTrue(this.testContextImplementation.CancellationTokenSource.IsCancellationRequested, "Not cancelled.."); + }); + } + + [TestMethodV1] + public void TestMethodInfoInvokeShouldFailOnTokenSourceCancellation() + { + var testablePlatformServiceProvider = new TestablePlatformServiceProvider(); + this.RunWithTestablePlatformService(testablePlatformServiceProvider, () => + { + testablePlatformServiceProvider.MockThreadOperations.CallBase = true; + PlatformServiceProvider.Instance = testablePlatformServiceProvider; + + testablePlatformServiceProvider.MockThreadOperations.Setup( + to => to.Execute(It.IsAny(), It.IsAny(), It.IsAny())).Callback((Action action, int timeoOut, CancellationToken cancelToken) => + { + try + { + Task.WaitAny(new[] { Task.Delay(100000) }, cancelToken); + } + catch (OperationCanceledException) + { + } + }); + + this.testMethodOptions.Timeout = 100000; + this.testContextImplementation.CancellationTokenSource.CancelAfter(100); + var method = new TestMethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions); + var result = method.Invoke(null); + + Assert.AreEqual(UTF.UnitTestOutcome.Timeout, result.Outcome); + StringAssert.Contains(result.TestFailureException.Message, "execution has been aborted"); + Assert.IsTrue(this.testContextImplementation.CancellationTokenSource.IsCancellationRequested, "Not cancelled.."); + }); + } + #endregion #region helper methods @@ -1256,9 +1308,9 @@ private void RunWithTestablePlatformService(TestablePlatformServiceProvider test try { testablePlatformServiceProvider.MockThreadOperations. - Setup(tho => tho.Execute(It.IsAny(), It.IsAny())). + Setup(tho => tho.Execute(It.IsAny(), It.IsAny(), It.IsAny())). Returns(true). - Callback((Action a, int timeout) => + Callback((Action a, int timeout, CancellationToken token) => { a.Invoke(); }); diff --git a/test/UnitTests/PlatformServices.Desktop.Unit.Tests/Services/DesktopThreadOperationsTests.cs b/test/UnitTests/PlatformServices.Desktop.Unit.Tests/Services/DesktopThreadOperationsTests.cs index a3d077fc4e..17b131cc43 100644 --- a/test/UnitTests/PlatformServices.Desktop.Unit.Tests/Services/DesktopThreadOperationsTests.cs +++ b/test/UnitTests/PlatformServices.Desktop.Unit.Tests/Services/DesktopThreadOperationsTests.cs @@ -6,11 +6,12 @@ namespace MSTestAdapter.PlatformServices.Desktop.UnitTests.Services extern alias FrameworkV1; using System; + using System.Diagnostics; using System.Reflection; using System.Threading; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; - + using Microsoft.VisualStudio.TestPlatform.ObjectModel; using MSTestAdapter.TestUtilities; using Assert = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.Assert; @@ -31,10 +32,13 @@ public DesktopThreadOperationsTests() public void ExecuteShouldRunActionOnANewThread() { int actionThreadID = 0; - Action action = () => { actionThreadID = Thread.CurrentThread.ManagedThreadId; }; - - Assert.IsTrue(this.asyncOperations.Execute(action, 1000)); + var cancellationTokenSource = new CancellationTokenSource(); + void action() + { + actionThreadID = Thread.CurrentThread.ManagedThreadId; + } + Assert.IsTrue(this.asyncOperations.Execute(action, 1000, cancellationTokenSource.Token)); Assert.AreNotEqual(Thread.CurrentThread.ManagedThreadId, actionThreadID); } @@ -44,24 +48,25 @@ public void ExecuteShouldKillTheThreadExecutingAsyncOnTimeout() ManualResetEvent timeoutMutex = new ManualResetEvent(false); var hasReachedEnd = false; var isThreadAbortThrown = false; + var cancellationTokenSource = new CancellationTokenSource(); - Action action = () => + void action() + { + try + { + timeoutMutex.WaitOne(); + hasReachedEnd = true; + } + catch (ThreadAbortException) { - try - { - timeoutMutex.WaitOne(); - hasReachedEnd = true; - } - catch (ThreadAbortException) - { - isThreadAbortThrown = true; - - // Resetting abort because there is a warning being thrown in the tests pane. - Thread.ResetAbort(); - } - }; - - Assert.IsFalse(this.asyncOperations.Execute(action, 1)); + isThreadAbortThrown = true; + + // Resetting abort because there is a warning being thrown in the tests pane. + Thread.ResetAbort(); + } + } + + Assert.IsFalse(this.asyncOperations.Execute(action, 1, cancellationTokenSource.Token)); timeoutMutex.Set(); Assert.IsFalse(hasReachedEnd); @@ -74,15 +79,15 @@ public void ExecuteShouldSpwanOfAthreadWithSpecificAttributes() var name = string.Empty; var apartmentState = ApartmentState.Unknown; var isBackground = false; + var cancellationTokenSource = new CancellationTokenSource(); + void action() + { + name = Thread.CurrentThread.Name; + apartmentState = Thread.CurrentThread.GetApartmentState(); + isBackground = Thread.CurrentThread.IsBackground; + } - Action action = () => - { - name = Thread.CurrentThread.Name; - apartmentState = Thread.CurrentThread.GetApartmentState(); - isBackground = Thread.CurrentThread.IsBackground; - }; - - Assert.IsTrue(this.asyncOperations.Execute(action, 100)); + Assert.IsTrue(this.asyncOperations.Execute(action, 100, cancellationTokenSource.Token)); Assert.AreEqual("MSTestAdapter Thread", name); Assert.AreEqual(Thread.CurrentThread.GetApartmentState(), apartmentState); @@ -100,5 +105,33 @@ public void ExecuteWithAbortSafetyShouldCatchThreadAbortExceptionsAndResetAbort( Assert.AreEqual(typeof(TargetInvocationException), exception.GetType()); Assert.AreEqual(typeof(ThreadAbortException), exception.InnerException.GetType()); } + + [TestMethod] + public void TokenCancelShouldAbortExecutingAction() + { + // setup + var cancellationTokenSource = new CancellationTokenSource(); + + // act + cancellationTokenSource.CancelAfter(100); + var result = this.asyncOperations.Execute(() => { Thread.Sleep(10000); }, 100000, cancellationTokenSource.Token); + + // validate + Assert.IsFalse(result, "The execution failed to abort"); + } + + [TestMethod] + public void TokenCancelShouldAbortIfAlreadyCancelled() + { + // setup + var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + + // act + var result = this.asyncOperations.Execute(() => { Thread.Sleep(10000); }, 100000, cancellationTokenSource.Token); + + // validate + Assert.IsFalse(result, "The execution failed to abort"); + } } } diff --git a/test/UnitTests/PlatformServices.Shared.Unit.Tests/netstandard1.0/ns10ThreadOperationsTests.cs b/test/UnitTests/PlatformServices.Shared.Unit.Tests/netstandard1.0/ns10ThreadOperationsTests.cs index 96861be632..10f9135e75 100644 --- a/test/UnitTests/PlatformServices.Shared.Unit.Tests/netstandard1.0/ns10ThreadOperationsTests.cs +++ b/test/UnitTests/PlatformServices.Shared.Unit.Tests/netstandard1.0/ns10ThreadOperationsTests.cs @@ -35,26 +35,36 @@ public ThreadOperationsTests() public void ExecuteShouldStartTheActionOnANewThread() { int actionThreadID = 0; - Action action = () => { actionThreadID = Thread.CurrentThread.ManagedThreadId; }; - - Assert.IsTrue(this.asyncOperations.Execute(action, 1000)); + void action() + { + actionThreadID = Thread.CurrentThread.ManagedThreadId; + } + CancellationTokenSource tokenSource = new CancellationTokenSource(); + Assert.IsTrue(this.asyncOperations.Execute(action, 1000, tokenSource.Token)); Assert.AreNotEqual(Thread.CurrentThread.ManagedThreadId, actionThreadID); } [TestMethod] public void ExecuteShouldReturnFalseIftheActionTimesout() { - Action action = () => { Task.Delay(100).Wait(); }; + void action() + { + Task.Delay(100).Wait(); + } - Assert.IsFalse(this.asyncOperations.Execute(action, 1)); + CancellationTokenSource tokenSource = new CancellationTokenSource(); + Assert.IsFalse(this.asyncOperations.Execute(action, 1, tokenSource.Token)); } [TestMethod] public void ExecuteWithAbortSafetyShouldInvokeTheAction() { var isInvoked = false; - Action action = () => { isInvoked = true; }; + void action() + { + isInvoked = true; + } this.asyncOperations.ExecuteWithAbortSafety(action);