feat!: [junit] capture test case level out, error and attachments

docs/circleci-recommendation.md

@@ -4,13 +4,13 @@
 
 ```yml
 - run:
-   name: Run Tests
-   command: |
-     dotnet test ... --logger:"junit;LogFilePath=TestResults/test-result.xml;FailureBodyFormat=Verbose;MethodFormat=Class"
+    name: Run Tests
+    command: |
+      dotnet test ... --logger:"junit;LogFilePath=TestResults/test-result.xml;FailureBodyFormat=Verbose;MethodFormat=Class"
 - store_test_results:
-   path: TestResults/
+    path: TestResults/
 - store_artifacts:
-   path: TestResults/
+    path: TestResults/
 ```
 
 See also: [this sample repository][circleci-windows-project-example].
@@ -75,7 +75,7 @@ the files must be stored using the
 
 ```yml
 - store_test_results:
-   path: TestResults/
+    path: TestResults/
 ```
 
 ### Store Artifacts
@@ -88,7 +88,7 @@ which is useful where the user requires information from the file that CircleCI
 
 ```yml
 - store_artifacts:
-   path: TestResults/
+    path: TestResults/
 ```
 
 ### Test Splitting
@@ -102,7 +102,7 @@ This technique reduces the total (elapsed-time) duration of the tests and thus p
 For an example of a CircleCI Windows project that demonstrates parallel test execution, take a look at
 [this sample repository][circleci-windows-project-example].
 
------
+---
 
 ## Footnote
 
@@ -112,7 +112,6 @@ As of July 20 2023:
   - e.g. `<system-out>` from within a `<testcase>` is shown.
 - Data outside of a `<testcase>` element is not interpreted by CircleCI.
 - The `<properties>` from a `<testcase>` aren't interpreted by CircleCI.
-- By default, the logger only puts console text (`<system-out>` and `<system-err>`) in elements at the `<testsuite>` level (not into `<testcase>`) so CircleCI  does not interpret it.
 
 e.g.
 

src/JUnit.Xml.Package/README.md

@@ -6,9 +6,9 @@ JUnit xml report extension for [Visual Studio Test Platform](https://github.com/
 
 ## Packages
 
-| Logger | Stable Package | Pre-release Package |
-| ------ | -------------- | ------------------- |
-| JUnit | [![NuGet](https://img.shields.io/nuget/v/JUnitXml.TestLogger.svg)](https://www.nuget.org/packages/JUnitXml.TestLogger/) | [![MyGet Pre Release](https://img.shields.io/myget/spekt/vpre/junitxml.testlogger.svg)](https://www.myget.org/feed/spekt/package/nuget/JunitXml.TestLogger) |
+| Logger | Stable Package                                                                                                          | Pre-release Package                                                                                                                                         |
+| ------ | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| JUnit  | [![NuGet](https://img.shields.io/nuget/v/JUnitXml.TestLogger.svg)](https://www.nuget.org/packages/JUnitXml.TestLogger/) | [![MyGet Pre Release](https://img.shields.io/myget/spekt/vpre/junitxml.testlogger.svg)](https://www.myget.org/feed/spekt/package/nuget/JunitXml.TestLogger) |
 
 If you're looking for `Nunit` or `Xunit` loggers, please see <https://github.com/spekt/testlogger>.
 
@@ -55,7 +55,7 @@ Platform Specific Recommendations:
 - [Jenkins Recommendation](/docs/jenkins-recommendation.md)
 - [CircleCI Recommendation](/docs/circleci-recommendation.md)
 
-After the logger name, command line arguments are provided as key/value pairs with the following general format. **Note** the quotes are required and key names are case sensitive.
+After the logger name, command line arguments are provided as key/value pairs with the following general format. **Note** the quotes are required, and key names are case-sensitive.
 
 ```none
 > dotnet test --test-adapter-path:. --logger:"junit;key1=value1;key2=value2"
@@ -84,6 +84,30 @@ We recommend this option for [GitLab](/docs/gitlab-recommendation.md) and [Circl
 - FailureBodyFormat=Default
 - FailureBodyFormat=Verbose
 
+#### StoreConsoleOutput
+
+You can use `StoreConsoleOutput` option to disable any `system-out` and `system-err` logs in both `testsuite`
+and `testcase` elements. By default, all console outputs are captured. Example usage:
+
+`dotnet test --logger:"junit;StoreConsoleOutput=false"`
+
+NOTE: test attachments are always emitted in `system-out` even when above option is false.
+
+**v5.x and later behavior**
+
+In v5.x and later, `system-out` and `system-err` logs are reported at a per `testcase` element in the report. Because each test
+adapter determines to also emit the console output and errors at test suite level, there may be some duplication for the messages.
+
+**v4.x and earlier behavior**
+
+Prior to v5.x, console stdout and stderr was reported only in the `system-out` and `system-err` elements at `testsuite` level. This
+would concatenate messages from all test results and information messages from adapter.
+
+##### Allowed Values
+
+- StoreConsoleOutput=true (default)
+- StoreConsoleOutput=false
+
 ## License
 
 MIT

src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs

@@ -200,14 +200,6 @@ private XElement CreateTestSuiteElement(List<TestResultInfo> results, TestRunCon
             var testCaseElements = results.Select(a => this.CreateTestCaseElement(a));
 
             StringBuilder stdOut = new StringBuilder();
-            foreach (var m in results.SelectMany(x => x.Messages))
-            {
-                if (TestResultMessage.StandardOutCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase))
-                {
-                    stdOut.AppendLine(m.Text);
-                }
-            }
-
             var frameworkInfo = messages.Where(x => x.Level == TestMessageLevel.Informational);
             if (frameworkInfo.Any())
             {
@@ -314,6 +306,43 @@ private XElement CreateTestCaseElement(TestResultInfo result)
                 testcaseElement.Add(skippedElement);
             }
 
+            // Add stdout and stderr to the testcase element
+            StringBuilder stdOut = new StringBuilder();
+            StringBuilder stdErr = new StringBuilder();
+            if (this.StoreConsoleOutputOption)
+            {
+                // Store the system-out and system-err only if store console output is enabled
+                foreach (var m in result.Messages)
+                {
+                    if (TestResultMessage.StandardOutCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase))
+                    {
+                        stdOut.AppendLine(m.Text);
+                    }
+                    else if (TestResultMessage.StandardErrorCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase))
+                    {
+                        stdErr.AppendLine(m.Text);
+                    }
+                }
+            }
+
+            // Attachments are always included in system-out irrespective of the StoreConsoleOutput option.
+            // See attachments spec here: https://plugins.jenkins.io/junit-attachments/
+            foreach (var attachment in result.Attachments)
+            {
+                // Ignore Attachment descriptions, it is not clear if CI systems use it.
+                stdOut.AppendLine($"[[ATTACHMENT|{attachment.FilePath}]]");
+            }
+
+            if (!string.IsNullOrWhiteSpace(stdOut.ToString()))
+            {
+                testcaseElement.Add(new XElement("system-out", new XCData(stdOut.ToString())));
+            }
+
+            if (!string.IsNullOrWhiteSpace(stdErr.ToString()))
+            {
+                testcaseElement.Add(new XElement("system-err", new XCData(stdErr.ToString())));
+            }
+
             return testcaseElement;
         }
 

test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs

@@ -93,7 +93,6 @@ public void TestResultFileShouldContainTestCases()
                 .ToList();
 
             Assert.AreEqual(15, failures.Count());
-            Assert.IsTrue(failures.All(x => x.Descendants().Count() == 1));
             Assert.IsTrue(failures.All(x => x.Descendants().First().Attribute("type").Value == "failure"));
 
             // Check failures

test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerStoreConsoleOutputOptionsAcceptanceTests.cs

@@ -41,6 +41,8 @@ public void StoreConsoleOutput_Default_ContainsConsoleOut()
             Assert.IsTrue(node.Value.Contains("{998AC9EC-7429-42CD-AD55-72037E7AF3D8}"));
             Assert.IsTrue(node.Value.Contains("{EEEE1DA6-6296-4486-BDA5-A50A19672F0F}"));
             Assert.IsTrue(node.Value.Contains("{C33FF4B5-75E1-4882-B968-DF9608BFE7C2}"));
+
+            this.TestCaseShouldHaveStdoutAndAttachment(resultsXml, "PassTest11");
         }
 
         [TestMethod]
@@ -77,6 +79,8 @@ public void StoreConsoleOutput_True_ContainsConsoleOut()
             Assert.IsTrue(node.Value.Contains("{998AC9EC-7429-42CD-AD55-72037E7AF3D8}"));
             Assert.IsTrue(node.Value.Contains("{EEEE1DA6-6296-4486-BDA5-A50A19672F0F}"));
             Assert.IsTrue(node.Value.Contains("{C33FF4B5-75E1-4882-B968-DF9608BFE7C2}"));
+
+            this.TestCaseShouldHaveStdoutAndAttachment(resultsXml, "PassTest11");
         }
 
         [TestMethod]
@@ -110,6 +114,8 @@ public void StoreConsoleOutput_False_ContainsConsoleOut()
             var node = resultsXml.XPathSelectElement("/testsuites/testsuite/system-out");
 
             Assert.IsTrue(node.Value.Equals(string.Empty));
+
+            this.TestCaseShouldHaveStdoutAndAttachment(resultsXml, "PassTest11");
         }
 
         [TestMethod]
@@ -127,5 +133,12 @@ public void StoreConsoleOutput_False_ContainsConsoleErr()
 
             Assert.IsTrue(node.Value.Equals(string.Empty));
         }
+
+        private void TestCaseShouldHaveStdoutAndAttachment(XDocument resultsXml, string testcaseName)
+        {
+            var node = resultsXml.XPathSelectElement($"/testsuites/testsuite/*[@name='{testcaseName}']/system-out");
+
+            Assert.IsTrue(node.Value.Contains("[[ATTACHMENT|"));
+        }
     }
 }

test/assets/JUnit.Xml.TestLogger.NetCore.Tests/UnitTest1.cs

@@ -1,4 +1,5 @@
 using System;
+using System.IO;
 using System.Threading.Tasks;
 using NUnit.Framework;
 
@@ -13,6 +14,14 @@ public async Task PassTest11()
         {
             Console.WriteLine("{2010CAE3-7BC0-4841-A5A3-7D5F947BB9FB}");
             Console.WriteLine("{998AC9EC-7429-42CD-AD55-72037E7AF3D8}");
+
+            var file = Path.Combine(Path.GetTempPath(), "y.txt");
+            if (!File.Exists(file))
+            {
+                File.Create(file);
+            }
+            TestContext.AddTestAttachment(file, "description");
+
             await Task.Delay(TimeSpan.FromMilliseconds(400));
         }
 
@@ -23,7 +32,7 @@ public void FailTest11()
             Console.WriteLine("{C33FF4B5-75E1-4882-B968-DF9608BFE7C2}");
             Console.Error.WriteLine("{D46DFA10-EEDD-49E5-804D-FE43051331A7}");
             Console.Error.WriteLine("{33F5FD22-6F40-499D-98E4-481D87FAEAA1}");
-         
+
             Assert.False(true);
         }