Skip to content

Commit

Permalink
Added StopOnError #675
Browse files Browse the repository at this point in the history
  • Loading branch information
Terje Sandstrom committed Jan 29, 2020
1 parent a819c5f commit c6075b6
Show file tree
Hide file tree
Showing 6 changed files with 57 additions and 20 deletions.
2 changes: 1 addition & 1 deletion build.cake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ var configuration = Argument("configuration", "Release");
//////////////////////////////////////////////////////////////////////

var version = "3.17.0";
var modifier = "-a16";
var modifier = "";

var dbgSuffix = configuration.ToLower() == "debug" ? "-dbg" : "";
var packageVersion = version + modifier + dbgSuffix;
Expand Down
3 changes: 3 additions & 0 deletions src/NUnitTestAdapter/AdapterSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public interface IAdapterSettings
bool UseNUnitIdforTestCaseId { get; }

int ConsoleOut { get; }
bool StopOnError { get; }

void Load(IDiscoveryContext context);
void Load(string settingsXml);
Expand Down Expand Up @@ -197,6 +198,7 @@ public AdapterSettings(TestLogger logger)
public bool UseParentFQNForParametrizedTests { get; private set; } // Default is false. True can fix certain test name patterns, but may have side effects.
public bool UseNUnitIdforTestCaseId { get; private set; } // default is false.
public int ConsoleOut { get; private set; }
public bool StopOnError { get; private set; }


public VsTestCategoryType VsTestCategoryType { get; private set; } = VsTestCategoryType.NUnit;
Expand Down Expand Up @@ -283,6 +285,7 @@ public void Load(string settingsXml)
UseParentFQNForParametrizedTests = GetInnerTextAsBool(nunitNode, nameof(UseParentFQNForParametrizedTests), false);
UseNUnitIdforTestCaseId = GetInnerTextAsBool(nunitNode, nameof(UseNUnitIdforTestCaseId), false);
ConsoleOut = GetInnerTextAsInt(nunitNode, nameof(ConsoleOut), 1); // 0 no output to console, 1 : output to console
StopOnError = GetInnerTextAsBool(nunitNode, nameof(StopOnError), false);
DumpXmlTestDiscovery = GetInnerTextAsBool(nunitNode, nameof(DumpXmlTestDiscovery), false);
DumpXmlTestResults = GetInnerTextAsBool(nunitNode, nameof(DumpXmlTestResults), false);
PreFilter = GetInnerTextAsBool(nunitNode, nameof(PreFilter), false);
Expand Down
36 changes: 26 additions & 10 deletions src/NUnitTestAdapter/NUnit3TestExecutor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,19 @@

namespace NUnit.VisualStudio.TestAdapter
{
public interface INUnit3TestExecutor
{
void StopRun();
IDumpXml Dump { get; }
IAdapterSettings Settings { get; }
}


[ExtensionUri(ExecutorUri)]
public sealed class NUnit3TestExecutor : NUnitTestAdapter, ITestExecutor, IDisposable
public sealed class NUnit3TestExecutor : NUnitTestAdapter, ITestExecutor, IDisposable, INUnit3TestExecutor
{
private DumpXml executionDumpXml;

public NUnit3TestExecutor()
{
EmbeddedAssemblyResolution.EnsureInitialized();
Expand Down Expand Up @@ -162,8 +172,7 @@ public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrame

void ITestExecutor.Cancel()
{
// _activeRunner?.StopRun(true);
NUnitEngineAdapter?.StopRun();
StopRun();
}

#endregion
Expand Down Expand Up @@ -222,20 +231,20 @@ private void RunAssembly(string assemblyPath, IGrouping<string, TestCase> testCa
// No need to restore if the seed was in runsettings file
if (!Settings.RandomSeedSpecified)
Settings.RestoreRandomSeed(Path.GetDirectoryName(assemblyPath));
DumpXml dumpXml = null;
executionDumpXml = null;
if (Settings.DumpXmlTestResults)
{
dumpXml = new DumpXml(assemblyPath);
executionDumpXml = new DumpXml(assemblyPath);
}

try
{
var package = CreateTestPackage(assemblyPath, testCases);
NUnitEngineAdapter.CreateRunner(package);
CreateTestOutputFolder();
dumpXml?.AddString($"<NUnitDiscoveryInExecution>{assemblyPath}</NUnitExecution>\r\n\r\n");
executionDumpXml?.AddString($"<NUnitDiscoveryInExecution>{assemblyPath}</NUnitExecution>\r\n\r\n");
var discoveryResults = NUnitEngineAdapter.Explore(filter); // _activeRunner.Explore(filter);
dumpXml?.AddString(discoveryResults.AsString());
executionDumpXml?.AddString(discoveryResults.AsString());

if (discoveryResults.IsRunnable)
{
Expand Down Expand Up @@ -268,8 +277,8 @@ private void RunAssembly(string assemblyPath, IGrouping<string, TestCase> testCa
TestLog.Info(" Skipping assembly - no matching test cases found");
return;
}
dumpXml?.AddString($"<NUnitExecution>{assemblyPath}</NUnitExecution>\r\n");
using (var listener = new NUnitEventListener(FrameworkHandle, testConverter, dumpXml, Settings))
executionDumpXml?.AddString($"<NUnitExecution>{assemblyPath}</NUnitExecution>\r\n");
using (var listener = new NUnitEventListener(FrameworkHandle, testConverter, this))
{
try
{
Expand Down Expand Up @@ -318,7 +327,7 @@ private void RunAssembly(string assemblyPath, IGrouping<string, TestCase> testCa
}
finally
{
dumpXml?.Dump4Execution();
executionDumpXml?.Dump4Execution();
try
{
NUnitEngineAdapter?.CloseRunner();
Expand Down Expand Up @@ -386,5 +395,12 @@ private void CreateTestOutputFolder()
}

#endregion

public void StopRun()
{
NUnitEngineAdapter?.StopRun();
}

public IDumpXml Dump => executionDumpXml;
}
}
15 changes: 11 additions & 4 deletions src/NUnitTestAdapter/NUnitEventListener.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,15 @@ public override object InitializeLifetimeService()
}
#endif

public NUnitEventListener(ITestExecutionRecorder recorder, ITestConverter testConverter, IDumpXml dumpXml, IAdapterSettings settings)
private INUnit3TestExecutor executor;

public NUnitEventListener(ITestExecutionRecorder recorder, ITestConverter testConverter, INUnit3TestExecutor executor)
{
this.dumpXml = dumpXml;
_settings = settings;
this.executor = executor;
dumpXml = executor.Dump;
_settings = executor.Settings;
_recorder = recorder;
_testConverter = testConverter;
_settings = settings;
}

#region ITestEventListener
Expand Down Expand Up @@ -167,6 +169,11 @@ public void TestFinished(NUnitTestEventTestCase resultNode)
{
_recorder.RecordResult(vsResult);
}

if (result.TestCaseResult.Outcome == TestOutcome.Failed && _settings.StopOnError)
{
executor.StopRun();
}
}

public void SuiteFinished(NUnitTestEventSuiteFinished resultNode)
Expand Down
3 changes: 3 additions & 0 deletions src/NUnitTestAdapter/NUnitTestAdapter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,9 @@ protected TestPackage CreateTestPackage(string assemblyName, IGrouping<string, T
if (Settings.TestProperties.Count > 0)
SetTestParameters(package.Settings, Settings.TestProperties);

if (Settings.StopOnError)
package.Settings[PackageSettings.StopOnError] = true;

// Always run one assembly at a time in process in its own domain
package.Settings[PackageSettings.ProcessModel] = "InProcess";

Expand Down
18 changes: 13 additions & 5 deletions src/NUnitTestAdapterTests/NUnitEventListenerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,16 @@ public class NUnitEventListenerTests
private NUnitEventListener listener;
private FakeFrameworkHandle testLog;
private NUnitTestCase fakeTestNode;
private INUnit3TestExecutor executor;
private IAdapterSettings settings;

[SetUp]
public void SetUp()
{
testLog = new FakeFrameworkHandle();
var settings = Substitute.For<IAdapterSettings>();
settings = Substitute.For<IAdapterSettings>();
executor = Substitute.For<INUnit3TestExecutor>();
executor.Settings.Returns(settings);
settings.CollectSourceInformation.Returns(true);
using (var testConverter = new TestConverter(new TestLogger(new MessageLoggerStub()), FakeTestData.AssemblyPath, settings))
{
Expand All @@ -60,7 +64,7 @@ public void SetUp()
testConverter.ConvertTestCase(fakeTestNode);
Assert.NotNull(testConverter.GetCachedTestCase("123"));

listener = new NUnitEventListener(testLog, testConverter, null, settings);
listener = new NUnitEventListener(testLog, testConverter, executor);
}
}

Expand Down Expand Up @@ -202,6 +206,8 @@ public class NUnitEventListenerOutputTests
private ITestConverter converter;
private IDumpXml dumpxml;
private IAdapterSettings settings;
private INUnit3TestExecutor executor;


private const string TestOutputProgress =
@"<test-output stream='Progress' testid='0-1001' testname='Something.TestClass.Whatever'><![CDATA[Whatever
Expand All @@ -228,12 +234,14 @@ public void Setup()
converter = Substitute.For<ITestConverter>();
dumpxml = Substitute.For<IDumpXml>();
settings = Substitute.For<IAdapterSettings>();
executor = Substitute.For<INUnit3TestExecutor>();
executor.Settings.Returns(settings);
}

[Test]
public void ThatNormalTestOutputIsOutput()
{
var sut = new NUnitEventListener(recorder, converter, dumpxml, settings);
var sut = new NUnitEventListener(recorder, converter, executor);
sut.OnTestEvent(TestOutputProgress);
sut.OnTestEvent(TestFinish);

Expand All @@ -244,7 +252,7 @@ public void ThatNormalTestOutputIsOutput()
[Test]
public void ThatNormalTestOutputIsError()
{
var sut = new NUnitEventListener(recorder, converter, dumpxml, settings);
var sut = new NUnitEventListener(recorder, converter, executor);
sut.OnTestEvent(TestOutputError);
sut.OnTestEvent(TestFinish);

Expand All @@ -255,7 +263,7 @@ public void ThatNormalTestOutputIsError()
[Test]
public void ThatTestOutputWithWhiteSpaceIsNotOutput()
{
var sut = new NUnitEventListener(recorder, converter, dumpxml, settings);
var sut = new NUnitEventListener(recorder, converter, executor);

sut.OnTestEvent(BlankTestOutput);

Expand Down

0 comments on commit c6075b6

Please sign in to comment.