diff --git a/docs/assets/TestResults.xml b/docs/assets/TestResults.xml index d5a1656..2432c0c 100644 --- a/docs/assets/TestResults.xml +++ b/docs/assets/TestResults.xml @@ -1,129 +1,177 @@  - + - + - - + + --TearDown - at JUnit.Xml.TestLogger.NetFull.Tests.FailingTearDown.TearDown() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 182 - - - at JUnit.Xml.TestLogger.NetFull.Tests.FailingTestSetup.SetUp() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 167 - - - - - - - at JUnit.Xml.TestLogger.NetFull.Tests.ParametrizedTestCases.TestData(Int32 x, String s) in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 232 + at JUnit.Xml.TestLogger.NetFull.Tests.FailingTearDown.TearDown() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs:line 191 + + + at JUnit.Xml.TestLogger.NetFull.Tests.FailingTestSetup.SetUp() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs:line 176 + + + + + + + at JUnit.Xml.TestLogger.NetFull.Tests.ParametrizedTestCases.TestData(Int32 x, String s) in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs:line 241 + + + at JUnit.Xml.TestLogger.NetFull.Tests.ParametrizedTestCases.TestData(Int32 x, String s) in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs:line 241 + + + + + + at JUnit.Xml.TestLogger.NetFull.Tests.UnitTest1.FailTest11() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs:line 36 + + - - at JUnit.Xml.TestLogger.NetFull.Tests.ParametrizedTestCases.TestData(Int32 x, String s) in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 232 + + + - - - - - at JUnit.Xml.TestLogger.NetFull.Tests.UnitTest1.FailTest11() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 27 + + - + + + + + + + + + + - - - - - - - - - - - at JUnit.Xml.TestLogger.NetFull.Tests.UnitTest2.FailTest22() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 94 - - + + at JUnit.Xml.TestLogger.NetFull.Tests.UnitTest2.FailTest22() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs:line 103 + + + - - - + + + + - + - - + + --TearDown - at JUnit.Xml.TestLogger.Tests2.FailingTearDown.TearDown() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 137 + at JUnit.Xml.TestLogger.Tests2.FailingTearDown.TearDown() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest2.cs:line 137 + + + at JUnit.Xml.TestLogger.Tests2.FailingTestSetup.SetUp() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest2.cs:line 122 - - at JUnit.Xml.TestLogger.Tests2.FailingTestSetup.SetUp() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 122 + + + + + + at JUnit.Xml.TestLogger.Tests2.ParametrizedTestCases.TestData(Int32 x, String s) in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest2.cs:line 187 - - - - - - at JUnit.Xml.TestLogger.Tests2.ParametrizedTestCases.TestData(Int32 x, String s) in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 187 + + at JUnit.Xml.TestLogger.Tests2.ParametrizedTestCases.TestData(Int32 x, String s) in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest2.cs:line 187 - - at JUnit.Xml.TestLogger.Tests2.ParametrizedTestCases.TestData(Int32 x, String s) in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 187 + + + + + at JUnit.Xml.TestLogger.Tests2.UnitTest1.FailTest11() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest2.cs:line 20 + + + + - - - - - at JUnit.Xml.TestLogger.Tests2.UnitTest1.FailTest11() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 20 + + - + + - - - - - at JUnit.Xml.TestLogger.Tests2.UnitTest2.FailTest22() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 49 + + at JUnit.Xml.TestLogger.Tests2.UnitTest2.FailTest22() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest2.cs:line 49 - + + - - - + + + + - {EEEE1DA6-6296-4486-BDA5-A50A19672F0F} + + at UnitTests.Domain.Orders.Checkout.CreateCheckoutOrderTests.TheHandler.RebuildsManifestIfAnyPromotionHasLimitedUse() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest3.cs:line 16 + + +Test Framework Informational Messages: +NUnit Adapter 4.0.0.0: Test execution started +Running all tests in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/bin/Debug/netcoreapp3.1/JUnit.Xml.TestLogger.NetCore.Tests.dll + NUnit3TestExecutor discovered 53 of 53 NUnit test cases using Current Discovery mode, Non-Explicit run +{EEEE1DA6-6296-4486-BDA5-A50A19672F0F} {C33FF4B5-75E1-4882-B968-DF9608BFE7C2} +Ignored: ignore reason +Inconclusive: test inconclusive {2010CAE3-7BC0-4841-A5A3-7D5F947BB9FB} {998AC9EC-7429-42CD-AD55-72037E7AF3D8} - -Test Framework Informational Messages: -NUnit Adapter 3.10.0.21: Test execution started -Running all tests in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\bin\Debug\netcoreapp3.1\JUnit.Xml.TestLogger.NetCore.Tests.dll -NUnit3TestExecutor converted 52 of 52 NUnit test cases -NUnit Adapter 3.10.0.21: Test execution complete +IgnoredTest: ignore reason +WarningTest: Warning +Ignored: ignore reason +Inconclusive: test inconclusive +IgnoredTest: ignore reason +WarningTest: Warning +NUnit Adapter 4.0.0.0: Test execution complete - Warning - SetUp failed for test fixture JUnit.Xml.TestLogger.NetFull.Tests.FailingOneTimeSetUp + Warning - Setup failed for test fixture JUnit.Xml.TestLogger.NetFull.Tests.FailingOneTimeSetUp Warning - System.InvalidOperationException : Operation is not valid due to the current state of the object. -Warning - at JUnit.Xml.TestLogger.NetFull.Tests.FailingOneTimeSetUp.OneTimeSetUp() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 152 +StackTrace: at JUnit.Xml.TestLogger.NetFull.Tests.FailingOneTimeSetUp.OneTimeSetUp() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs:line 161 Warning - TearDown failed for test fixture JUnit.Xml.TestLogger.NetFull.Tests.FailingOneTimeTearDown Warning - TearDown : System.InvalidOperationException : Operation is not valid due to the current state of the object. -Warning - --TearDown - at JUnit.Xml.TestLogger.NetFull.Tests.FailingOneTimeTearDown.OneTimeTearDown() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 197 +StackTrace: --TearDown + at JUnit.Xml.TestLogger.NetFull.Tests.FailingOneTimeTearDown.OneTimeTearDown() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs:line 206 Warning - {D46DFA10-EEDD-49E5-804D-FE43051331A7} Warning - {33F5FD22-6F40-499D-98E4-481D87FAEAA1} -Warning - SetUp failed for test fixture JUnit.Xml.TestLogger.Tests2.FailingOneTimeSetUp +Warning - Setup failed for test fixture JUnit.Xml.TestLogger.Tests2.FailingOneTimeSetUp Warning - System.InvalidOperationException : Operation is not valid due to the current state of the object. -Warning - at JUnit.Xml.TestLogger.Tests2.FailingOneTimeSetUp.OneTimeSetUp() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 107 +StackTrace: at JUnit.Xml.TestLogger.Tests2.FailingOneTimeSetUp.OneTimeSetUp() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest2.cs:line 107 Warning - TearDown failed for test fixture JUnit.Xml.TestLogger.Tests2.FailingOneTimeTearDown Warning - TearDown : System.InvalidOperationException : Operation is not valid due to the current state of the object. -Warning - --TearDown - at JUnit.Xml.TestLogger.Tests2.FailingOneTimeTearDown.OneTimeTearDown() in C:\Users\REDACTED\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 152 +StackTrace: --TearDown + at JUnit.Xml.TestLogger.Tests2.FailingOneTimeTearDown.OneTimeTearDown() in /mnt/docs/src/spekt/testlogger/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest2.cs:line 152 \ No newline at end of file diff --git a/docs/circleci-recommendation.md b/docs/circleci-recommendation.md index 91d0f34..4ce9f5f 100644 --- a/docs/circleci-recommendation.md +++ b/docs/circleci-recommendation.md @@ -4,13 +4,13 @@ ```yml - run: - name: Run Tests - command: | - dotnet test ... --logger:"junit;LogFilePath=TestResults/test-result.xml;FailureBodyFormat=Verbose;MethodFormat=Class" + name: Run Tests + command: | + dotnet test ... --logger:"junit;LogFilePath=TestResults/test-result.xml;FailureBodyFormat=Verbose;MethodFormat=Class" - store_test_results: - path: TestResults/ + path: TestResults/ - store_artifacts: - path: TestResults/ + path: TestResults/ ``` See also: [this sample repository][circleci-windows-project-example]. @@ -75,7 +75,7 @@ the files must be stored using the ```yml - store_test_results: - path: TestResults/ + path: TestResults/ ``` ### Store Artifacts @@ -88,7 +88,7 @@ which is useful where the user requires information from the file that CircleCI ```yml - store_artifacts: - path: TestResults/ + path: TestResults/ ``` ### Test Splitting @@ -102,7 +102,7 @@ This technique reduces the total (elapsed-time) duration of the tests and thus p For an example of a CircleCI Windows project that demonstrates parallel test execution, take a look at [this sample repository][circleci-windows-project-example]. ------ +--- ## Footnote @@ -112,7 +112,6 @@ As of July 20 2023: - e.g. `` from within a `` is shown. - Data outside of a `` element is not interpreted by CircleCI. - The `` from a `` aren't interpreted by CircleCI. -- By default, the logger only puts console text (`` and ``) in elements at the `` level (not into ``) so CircleCI does not interpret it. e.g. diff --git a/src/JUnit.Xml.Package/README.md b/src/JUnit.Xml.Package/README.md index 19d3f93..6eda931 100644 --- a/src/JUnit.Xml.Package/README.md +++ b/src/JUnit.Xml.Package/README.md @@ -6,9 +6,9 @@ JUnit xml report extension for [Visual Studio Test Platform](https://github.com/ ## Packages -| Logger | Stable Package | Pre-release Package | -| ------ | -------------- | ------------------- | -| JUnit | [![NuGet](https://img.shields.io/nuget/v/JUnitXml.TestLogger.svg)](https://www.nuget.org/packages/JUnitXml.TestLogger/) | [![MyGet Pre Release](https://img.shields.io/myget/spekt/vpre/junitxml.testlogger.svg)](https://www.myget.org/feed/spekt/package/nuget/JunitXml.TestLogger) | +| Logger | Stable Package | Pre-release Package | +| ------ | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| JUnit | [![NuGet](https://img.shields.io/nuget/v/JUnitXml.TestLogger.svg)](https://www.nuget.org/packages/JUnitXml.TestLogger/) | [![MyGet Pre Release](https://img.shields.io/myget/spekt/vpre/junitxml.testlogger.svg)](https://www.myget.org/feed/spekt/package/nuget/JunitXml.TestLogger) | If you're looking for `Nunit` or `Xunit` loggers, please see . @@ -55,7 +55,7 @@ Platform Specific Recommendations: - [Jenkins Recommendation](/docs/jenkins-recommendation.md) - [CircleCI Recommendation](/docs/circleci-recommendation.md) -After the logger name, command line arguments are provided as key/value pairs with the following general format. **Note** the quotes are required and key names are case sensitive. +After the logger name, command line arguments are provided as key/value pairs with the following general format. **Note** the quotes are required, and key names are case-sensitive. ```none > dotnet test --test-adapter-path:. --logger:"junit;key1=value1;key2=value2" @@ -84,6 +84,30 @@ We recommend this option for [GitLab](/docs/gitlab-recommendation.md) and [Circl - FailureBodyFormat=Default - FailureBodyFormat=Verbose +#### StoreConsoleOutput + +You can use `StoreConsoleOutput` option to disable any `system-out` and `system-err` logs in both `testsuite` +and `testcase` elements. By default, all console outputs are captured. Example usage: + +`dotnet test --logger:"junit;StoreConsoleOutput=false"` + +NOTE: test attachments are always emitted in `system-out` even when above option is false. + +**v5.x and later behavior** + +In v5.x and later, `system-out` and `system-err` logs are reported at a per `testcase` element in the report. Because each test +adapter determines to also emit the console output and errors at test suite level, there may be some duplication for the messages. + +**v4.x and earlier behavior** + +Prior to v5.x, console stdout and stderr was reported only in the `system-out` and `system-err` elements at `testsuite` level. This +would concatenate messages from all test results and information messages from adapter. + +##### Allowed Values + +- StoreConsoleOutput=true (default) +- StoreConsoleOutput=false + ## License MIT diff --git a/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs b/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs index 10c994a..01fcbba 100644 --- a/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs +++ b/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs @@ -200,14 +200,6 @@ private XElement CreateTestSuiteElement(List results, TestRunCon var testCaseElements = results.Select(a => this.CreateTestCaseElement(a)); StringBuilder stdOut = new StringBuilder(); - foreach (var m in results.SelectMany(x => x.Messages)) - { - if (TestResultMessage.StandardOutCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase)) - { - stdOut.AppendLine(m.Text); - } - } - var frameworkInfo = messages.Where(x => x.Level == TestMessageLevel.Informational); if (frameworkInfo.Any()) { @@ -314,6 +306,43 @@ private XElement CreateTestCaseElement(TestResultInfo result) testcaseElement.Add(skippedElement); } + // Add stdout and stderr to the testcase element + StringBuilder stdOut = new StringBuilder(); + StringBuilder stdErr = new StringBuilder(); + if (this.StoreConsoleOutputOption) + { + // Store the system-out and system-err only if store console output is enabled + foreach (var m in result.Messages) + { + if (TestResultMessage.StandardOutCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase)) + { + stdOut.AppendLine(m.Text); + } + else if (TestResultMessage.StandardErrorCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase)) + { + stdErr.AppendLine(m.Text); + } + } + } + + // Attachments are always included in system-out irrespective of the StoreConsoleOutput option. + // See attachments spec here: https://plugins.jenkins.io/junit-attachments/ + foreach (var attachment in result.Attachments) + { + // Ignore Attachment descriptions, it is not clear if CI systems use it. + stdOut.AppendLine($"[[ATTACHMENT|{attachment.FilePath}]]"); + } + + if (!string.IsNullOrWhiteSpace(stdOut.ToString())) + { + testcaseElement.Add(new XElement("system-out", new XCData(stdOut.ToString()))); + } + + if (!string.IsNullOrWhiteSpace(stdErr.ToString())) + { + testcaseElement.Add(new XElement("system-err", new XCData(stdErr.ToString()))); + } + return testcaseElement; } diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs index e80ef91..d63142b 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs @@ -93,7 +93,6 @@ public void TestResultFileShouldContainTestCases() .ToList(); Assert.AreEqual(15, failures.Count()); - Assert.IsTrue(failures.All(x => x.Descendants().Count() == 1)); Assert.IsTrue(failures.All(x => x.Descendants().First().Attribute("type").Value == "failure")); // Check failures diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerStoreConsoleOutputOptionsAcceptanceTests.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerStoreConsoleOutputOptionsAcceptanceTests.cs index 7fca9ad..d679cc3 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerStoreConsoleOutputOptionsAcceptanceTests.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerStoreConsoleOutputOptionsAcceptanceTests.cs @@ -41,6 +41,8 @@ public void StoreConsoleOutput_Default_ContainsConsoleOut() Assert.IsTrue(node.Value.Contains("{998AC9EC-7429-42CD-AD55-72037E7AF3D8}")); Assert.IsTrue(node.Value.Contains("{EEEE1DA6-6296-4486-BDA5-A50A19672F0F}")); Assert.IsTrue(node.Value.Contains("{C33FF4B5-75E1-4882-B968-DF9608BFE7C2}")); + + this.TestCaseShouldHaveStdoutAndAttachment(resultsXml, "PassTest11"); } [TestMethod] @@ -77,6 +79,8 @@ public void StoreConsoleOutput_True_ContainsConsoleOut() Assert.IsTrue(node.Value.Contains("{998AC9EC-7429-42CD-AD55-72037E7AF3D8}")); Assert.IsTrue(node.Value.Contains("{EEEE1DA6-6296-4486-BDA5-A50A19672F0F}")); Assert.IsTrue(node.Value.Contains("{C33FF4B5-75E1-4882-B968-DF9608BFE7C2}")); + + this.TestCaseShouldHaveStdoutAndAttachment(resultsXml, "PassTest11"); } [TestMethod] @@ -110,6 +114,8 @@ public void StoreConsoleOutput_False_ContainsConsoleOut() var node = resultsXml.XPathSelectElement("/testsuites/testsuite/system-out"); Assert.IsTrue(node.Value.Equals(string.Empty)); + + this.TestCaseShouldHaveStdoutAndAttachment(resultsXml, "PassTest11"); } [TestMethod] @@ -127,5 +133,12 @@ public void StoreConsoleOutput_False_ContainsConsoleErr() Assert.IsTrue(node.Value.Equals(string.Empty)); } + + private void TestCaseShouldHaveStdoutAndAttachment(XDocument resultsXml, string testcaseName) + { + var node = resultsXml.XPathSelectElement($"/testsuites/testsuite/*[@name='{testcaseName}']/system-out"); + + Assert.IsTrue(node.Value.Contains("[[ATTACHMENT|")); + } } } diff --git a/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs b/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs index c4813c4..9ede2af 100644 --- a/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs +++ b/test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Threading.Tasks; using NUnit.Framework; @@ -13,6 +14,14 @@ public async Task PassTest11() { Console.WriteLine("{2010CAE3-7BC0-4841-A5A3-7D5F947BB9FB}"); Console.WriteLine("{998AC9EC-7429-42CD-AD55-72037E7AF3D8}"); + + var file = Path.Combine(Path.GetTempPath(), "y.txt"); + if (!File.Exists(file)) + { + File.Create(file); + } + TestContext.AddTestAttachment(file, "description"); + await Task.Delay(TimeSpan.FromMilliseconds(400)); } @@ -23,7 +32,7 @@ public void FailTest11() Console.WriteLine("{C33FF4B5-75E1-4882-B968-DF9608BFE7C2}"); Console.Error.WriteLine("{D46DFA10-EEDD-49E5-804D-FE43051331A7}"); Console.Error.WriteLine("{33F5FD22-6F40-499D-98E4-481D87FAEAA1}"); - + Assert.False(true); }