Add system-out and system-error to each testcase

spekt · Nov 15, 2021 · 507b9cb · 507b9cb
1 parent b655abe
commit 507b9cb
Show file tree

Hide file tree

Showing 3 changed files with 128 additions and 5 deletions.
diff --git a/README.md b/README.md
@@ -12,15 +12,15 @@ JUnit xml report extension for [Visual Studio Test Platform](https://github.com/
 | ------ | -------------- | ------------------- |
 | JUnit | [![NuGet](https://img.shields.io/nuget/v/JUnitXml.TestLogger.svg)](https://www.nuget.org/packages/JUnitXml.TestLogger/) | [![MyGet Pre Release](https://img.shields.io/myget/spekt/vpre/junitxml.testlogger.svg)](https://www.myget.org/feed/spekt/package/nuget/JunitXml.TestLogger) |
 
-If you're looking for `Nunit`, `Xunit` or `appveyor` loggers, visit following repositories:
+If you're looking for `NUnit`, `xUnit` or `appveyor` loggers, visit following repositories:
 
 - <https://github.com/spekt/nunit.testlogger>
 - <https://github.com/spekt/xunit.testlogger>
 - <https://github.com/spekt/appveyor.testlogger>
 
 ## Usage
 
-The JUnit Test Logger generates xml reports in the [Ant Junit Format](https://github.com/windyroad/JUnit-Schema), which the JUnit 5 repository refers to as the de-facto standard. While the generated xml complies with that schema, it does not contain values in every case. For example, the logger currently does not log any `properties`. Please [refer to a sample file](docs/assets/TestResults.xml) to see an example. If you find that the format is missing data required by your CI/CD system, please open an issue or PR.
+The JUnit Test Logger generates xml reports in the [Jenkins JUnit Format](https://github.com/junit-team/junit5/blob/main/platform-tests/src/test/resources/jenkins-junit.xsd), which the JUnit 5 repository refers to as the de-facto standard. While the generated xml complies with that schema, it does not contain values in every case. For example, the logger currently does not log any `properties`. Please [refer to a sample file](docs/assets/TestResults.xml) to see an example. If you find that the format is missing data required by your CI/CD system, please open an issue or PR.
 
 To use the logger, follow these steps:
 
@@ -47,6 +47,89 @@ test logs from over-writing eachother.
 
 [config-wiki]: https://github.com/spekt/testlogger/wiki/Logger-Configuration
 
+### Junit Output Notes
+
+The xml reports will have standard output (stdout) and standard error (stderr) from the test output. Stdout is logged on a per-testcase basis, but stderr
+
+#### NUnit
+
+On a testcase failure, NUnit will redirect stderr to stdout for each `TestCase`. Because of this, both stdout and stderr will be captured in the JUnit xml file within the `<system-out></system-out>` tags for each test case. It will also be capture within the `<system-out></system-out>` tags at the `<testsuite>` level.
+
+##### NUnit Example
+```csharp
+[TestCase]
+public void TestThatFailsWithStdErr()
+{
+    Console.WriteLine("Output to stderr", Console.Error);
+    Assert.Fail();
+}
+```
+Console Output
+```
+  Failed TestThatFailsWithStdErr() [71 ms]
+  Stack Trace:
+     at NUnit.UnitTests.Tests.TestThatFailsWithStdErr() in /home/runner/work/dotnet_test_output_reporting/dotnet_test_output_reporting/NUnit.UnitTests/UnitTest1.cs:line 42
+
+  Standard Output Messages:
+ Output to stderr
+```
+
+```xml
+<testcase classname="NUnit.UnitTests.Tests" name="TestThatFailsWithStdOutAndStdErr()" time="0.0004670">
+  <failure type="failure">at NUnit.UnitTests.Tests.TestThatFailsWithStdOutAndStdErr() in D:\Developer\dotnet_test_output_reporting\NUnit.UnitTests\UnitTest1.cs:line 50</failure>
+  <system-out>Output to stdout
+Output to stderr
+
+</system-out>
+</testcase>
+```
+
+#### xUnit
+
+On a testcase failure, by default xUnit will not log any stdout or stderr to the console. Stdout from each `Fact` will be captured in the JUnit xml within the `<system-out></system-out>` tags at the `testsuite` level.
+
+##### xUnit Example
+```csharp
+[Fact]
+public void TestThatFailsWithStdErr()
+{
+    Console.WriteLine("Output to stderr", Console.Error);
+    Assert.True(false, "test failure");
+}
+```
+```
+Expected: True
+Actual:   False
+  Stack Trace:
+     at XUnit.UnitTests.UnitTest1.TestThatFailsWithStdErr() in /home/runner/work/dotnet_test_output_reporting/dotnet_test_output_reporting/XUnit.UnitTests/UnitTest1.cs:line 41
+  Failed XUnit.UnitTests.UnitTest1.TestThatFailsWithStdOutAndStdErr [< 1 ms]
+  Error Message:
+   test failure
+```
+```xml
+<testcase classname="XUnit.UnitTests.UnitTest1" name="TestThatFailsWithStdErr" time="0.0030326">
+  <failure type="failure" message="test failure&#xD;&#xA;Expected: True&#xD;&#xA;Actual:   False">at XUnit.UnitTests.UnitTest1.TestThatFailsWithStdErr() in D:\Developer\dotnet_test_output_reporting\XUnit.UnitTests\UnitTest1.cs:line 41</failure>
+</testcase>
+```
+
+
+#### MsTest
+
+#### MsTest Example
+```csharp
+
+```
+```
+Failed TestThatFailsWithStdErr [< 1 ms]
+Error Message:
+ Assert.Fail failed. 
+Stack Trace:
+   at MsTest.UnitTests.UnitTest1.TestThatFailsWithStdErr() in /home/runner/work/dotnet_test_output_reporting/dotnet_test_output_reporting/MsTest.UnitTests/UnitTest1.cs:line 42
+
+Standard Output Messages:
+Output to stderr
+```
+
 ### Customizing Junit XML Contents
 
 There are several options to customize how the junit xml is populated. These options exist to

diff --git a/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs b/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs
@@ -307,6 +307,34 @@ private XElement CreateTestCaseElement(TestResultInfo result)
                 testcaseElement.Add(skippedElement);
             }
 
+            StringBuilder stdErr = new StringBuilder();
+            foreach (var m in result.Messages)
+            {
+                if (TestResultMessage.StandardErrorCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase))
+                {
+                    stdErr.AppendLine(m.Text);
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(stdErr.ToString()))
+            {
+                testcaseElement.Add(new XElement("system-err", stdErr.ToString()));
+            }
+
+            StringBuilder stdOut = new StringBuilder();
+            foreach (var m in result.Messages)
+            {
+                if (TestResultMessage.StandardOutCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase))
+                {
+                    stdOut.AppendLine(m.Text);
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(stdOut.ToString()))
+            {
+                testcaseElement.Add(new XElement("system-out", stdOut.ToString()));
+            }
+
             return testcaseElement;
         }
 

diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs
@@ -85,8 +85,7 @@ public void TestResultFileShouldContainTestCases()
                 .ToList();
 
             Assert.AreEqual(14, failures.Count());
-            Assert.IsTrue(failures.All(x => x.Descendants().Count() == 1));
-            Assert.IsTrue(failures.All(x => x.Descendants().First().Attribute("type").Value == "failure"));
+            Assert.IsTrue(failures.All(x => x.Descendants().Any(element => element.Attribute("type").Value == "failure")));
 
             // Check failures
             var skips = testcases
@@ -96,6 +95,19 @@ public void TestResultFileShouldContainTestCases()
             Assert.AreEqual(6, skips.Count());
         }
 
+        [TestMethod]
+        public void TestCasesShouldContainStandardOut()
+        {
+            var node = this.resultsXml.XPathSelectElements("/testsuites/testsuite").Descendants();
+            var testcases = node.Where(x => x.Name.LocalName == "testcase").ToList();
+
+            var testCasesWithSystemOut = testcases
+                .Where(x => x.Descendants().Any(y => y.Name.LocalName == "system-out"))
+                .ToList();
+
+            Assert.AreEqual(2, testCasesWithSystemOut.Count());
+        }
+
         [TestMethod]
         public void TestResultFileShouldContainStandardOut()
         {
@@ -108,7 +120,7 @@ public void TestResultFileShouldContainStandardOut()
         }
 
         [TestMethod]
-        public void TestResultFileShouldContainErrordOut()
+        public void TestResultFileShouldContainErrorOut()
         {
             var node = this.resultsXml.XPathSelectElement("/testsuites/testsuite/system-err");