From f03b68a55351b30348432e966b7e4703331ebe5a Mon Sep 17 00:00:00 2001 From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Date: Tue, 9 May 2023 18:48:49 +0300 Subject: [PATCH] Add options to include passed and skipped tests in the summary --- .../AnnotationFormatSpecs.cs | 20 +++ .../AnnotationSpecs.cs | 14 ++ .../InitializationSpecs.cs | 6 +- GitHubActionsTestLogger.Tests/SummarySpecs.cs | 140 ++++++++++++++++++ GitHubActionsTestLogger/TestLoggerContext.cs | 41 +++-- GitHubActionsTestLogger/TestLoggerOptions.cs | 15 +- GitHubActionsTestLogger/TestRunStatistics.cs | 21 ++- GitHubActionsTestLogger/TestSummary.cs | 72 ++++++--- Readme.md | 24 ++- 9 files changed, 315 insertions(+), 38 deletions(-) diff --git a/GitHubActionsTestLogger.Tests/AnnotationFormatSpecs.cs b/GitHubActionsTestLogger.Tests/AnnotationFormatSpecs.cs index 9880953..dc43f61 100644 --- a/GitHubActionsTestLogger.Tests/AnnotationFormatSpecs.cs +++ b/GitHubActionsTestLogger.Tests/AnnotationFormatSpecs.cs @@ -4,11 +4,17 @@ using GitHubActionsTestLogger.Tests.Utils.Extensions; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Xunit; +using Xunit.Abstractions; namespace GitHubActionsTestLogger.Tests; public class AnnotationFormatSpecs { + private readonly ITestOutputHelper _testOutput; + + public AnnotationFormatSpecs(ITestOutputHelper testOutput) => + _testOutput = testOutput; + [Fact] public void Custom_format_can_reference_test_name() { @@ -40,6 +46,8 @@ public void Custom_format_can_reference_test_name() output.Should().Contain(""); output.Should().Contain("[Test1]"); + + _testOutput.WriteLine(output); } [Fact] @@ -75,6 +83,8 @@ public void Custom_format_can_reference_test_traits() output.Should().Contain(" Test1>"); output.Should().Contain("[UI Test -> Test1]"); + + _testOutput.WriteLine(output); } [Fact] @@ -109,6 +119,8 @@ public void Custom_format_can_reference_test_error_message() output.Should().Contain(""); output.Should().Contain("[Test1: ErrorMessage]"); + + _testOutput.WriteLine(output); } [Fact] @@ -143,6 +155,8 @@ public void Custom_format_can_reference_test_error_stacktrace() output.Should().Contain(""); output.Should().Contain("[Test1: ErrorStackTrace]"); + + _testOutput.WriteLine(output); } [Fact] @@ -179,6 +193,8 @@ public void Custom_format_can_reference_test_target_framework_name() output.Should().Contain(""); output.Should().Contain("[Test1 (FakeTargetFramework)]"); + + _testOutput.WriteLine(output); } [Fact] @@ -210,6 +226,8 @@ public void Custom_format_can_contain_newlines() var output = commandWriter.ToString().Trim(); output.Should().Contain("foo%0Abar"); + + _testOutput.WriteLine(output); } [Fact] @@ -240,5 +258,7 @@ public void Default_format_references_test_name_and_error_message() output.Should().Contain("Test1"); output.Should().Contain("ErrorMessage"); + + _testOutput.WriteLine(output); } } \ No newline at end of file diff --git a/GitHubActionsTestLogger.Tests/AnnotationSpecs.cs b/GitHubActionsTestLogger.Tests/AnnotationSpecs.cs index a2afeae..cde0bfa 100644 --- a/GitHubActionsTestLogger.Tests/AnnotationSpecs.cs +++ b/GitHubActionsTestLogger.Tests/AnnotationSpecs.cs @@ -5,11 +5,17 @@ using GitHubActionsTestLogger.Tests.Utils.Extensions; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Xunit; +using Xunit.Abstractions; namespace GitHubActionsTestLogger.Tests; public class AnnotationSpecs { + private readonly ITestOutputHelper _testOutput; + + public AnnotationSpecs(ITestOutputHelper testOutput) => + _testOutput = testOutput; + [Fact] public void Passed_tests_do_not_get_reported() { @@ -95,6 +101,8 @@ public void Failed_tests_get_reported() output.Should().StartWith("::error "); output.Should().Contain("Test1"); output.Should().Contain("ErrorMessage"); + + _testOutput.WriteLine(output); } [Fact] @@ -146,6 +154,8 @@ at CliWrap.Tests.CancellationSpecs.I_can_execute_a_command_with_buffering_and_ca output.Should().Contain("line=75"); output.Should().Contain("I can execute a command with buffering and cancel it immediately"); output.Should().Contain("ErrorMessage"); + + _testOutput.WriteLine(output); } [Fact] @@ -201,6 +211,8 @@ at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotifi output.Should().Contain("line=187"); output.Should().Contain("SendEnvelopeAsync_ItemRateLimit_DropsItem"); output.Should().Contain("ErrorMessage"); + + _testOutput.WriteLine(output); } [Fact] @@ -234,5 +246,7 @@ public void Failed_tests_get_reported_with_approximate_source_information_if_exc output.Should().MatchRegex(@"file=.*?\.csproj"); output.Should().Contain("Test1"); output.Should().Contain("ErrorMessage"); + + _testOutput.WriteLine(output); } } \ No newline at end of file diff --git a/GitHubActionsTestLogger.Tests/InitializationSpecs.cs b/GitHubActionsTestLogger.Tests/InitializationSpecs.cs index 751ba01..2322ba0 100644 --- a/GitHubActionsTestLogger.Tests/InitializationSpecs.cs +++ b/GitHubActionsTestLogger.Tests/InitializationSpecs.cs @@ -33,7 +33,9 @@ public void Logger_can_be_used_with_custom_configuration() var parameters = new Dictionary { ["annotations.titleFormat"] = "TitleFormat", - ["annotations.messageFormat"] = "MessageFormat" + ["annotations.messageFormat"] = "MessageFormat", + ["summary.includePassedTests"] = "true", + ["summary.includeSkippedTests"] = "true" }; // Act @@ -43,5 +45,7 @@ public void Logger_can_be_used_with_custom_configuration() logger.Context.Should().NotBeNull(); logger.Context?.Options.AnnotationTitleFormat.Should().Be("TitleFormat"); logger.Context?.Options.AnnotationMessageFormat.Should().Be("MessageFormat"); + logger.Context?.Options.SummaryIncludePassedTests.Should().BeTrue(); + logger.Context?.Options.SummaryIncludeSkippedTests.Should().BeTrue(); } } \ No newline at end of file diff --git a/GitHubActionsTestLogger.Tests/SummarySpecs.cs b/GitHubActionsTestLogger.Tests/SummarySpecs.cs index f6827dd..ed44101 100644 --- a/GitHubActionsTestLogger.Tests/SummarySpecs.cs +++ b/GitHubActionsTestLogger.Tests/SummarySpecs.cs @@ -4,11 +4,17 @@ using GitHubActionsTestLogger.Tests.Utils.Extensions; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Xunit; +using Xunit.Abstractions; namespace GitHubActionsTestLogger.Tests; public class SummarySpecs { + private readonly ITestOutputHelper _testOutput; + + public SummarySpecs(ITestOutputHelper testOutput) => + _testOutput = testOutput; + [Fact] public void Test_summary_contains_test_suite_name() { @@ -30,6 +36,114 @@ public void Test_summary_contains_test_suite_name() var output = summaryWriter.ToString().Trim(); output.Should().Contain("TestProject"); + + _testOutput.WriteLine(output); + } + + [Fact] + public void Test_summary_contains_names_of_passed_tests_if_enabled() + { + // Arrange + using var summaryWriter = new StringWriter(); + + var context = new TestLoggerContext( + new GitHubWorkflow( + TextWriter.Null, + summaryWriter + ), + new TestLoggerOptions + { + SummaryIncludePassedTests = true + } + ); + + // Act + context.SimulateTestRun( + new TestResultBuilder() + .SetDisplayName("Test1") + .SetFullyQualifiedName("TestProject.SomeTests.Test1") + .SetOutcome(TestOutcome.Passed) + .Build(), + new TestResultBuilder() + .SetDisplayName("Test2") + .SetFullyQualifiedName("TestProject.SomeTests.Test2") + .SetOutcome(TestOutcome.Passed) + .Build(), + new TestResultBuilder() + .SetDisplayName("Test3") + .SetFullyQualifiedName("TestProject.SomeTests.Test3") + .SetOutcome(TestOutcome.Passed) + .Build(), + new TestResultBuilder() + .SetDisplayName("Test4") + .SetFullyQualifiedName("TestProject.SomeTests.Test4") + .SetOutcome(TestOutcome.Failed) + .SetErrorMessage("ErrorMessage4") + .Build() + ); + + // Assert + var output = summaryWriter.ToString().Trim(); + + output.Should().Contain("Test1"); + output.Should().Contain("Test2"); + output.Should().Contain("Test3"); + output.Should().Contain("Test4"); + + _testOutput.WriteLine(output); + } + + [Fact] + public void Test_summary_contains_names_of_skipped_tests_if_enabled() + { + // Arrange + using var summaryWriter = new StringWriter(); + + var context = new TestLoggerContext( + new GitHubWorkflow( + TextWriter.Null, + summaryWriter + ), + new TestLoggerOptions + { + SummaryIncludeSkippedTests = true + } + ); + + // Act + context.SimulateTestRun( + new TestResultBuilder() + .SetDisplayName("Test1") + .SetFullyQualifiedName("TestProject.SomeTests.Test1") + .SetOutcome(TestOutcome.Skipped) + .Build(), + new TestResultBuilder() + .SetDisplayName("Test2") + .SetFullyQualifiedName("TestProject.SomeTests.Test2") + .SetOutcome(TestOutcome.Skipped) + .Build(), + new TestResultBuilder() + .SetDisplayName("Test3") + .SetFullyQualifiedName("TestProject.SomeTests.Test3") + .SetOutcome(TestOutcome.Skipped) + .Build(), + new TestResultBuilder() + .SetDisplayName("Test4") + .SetFullyQualifiedName("TestProject.SomeTests.Test4") + .SetOutcome(TestOutcome.Failed) + .SetErrorMessage("ErrorMessage4") + .Build() + ); + + // Assert + var output = summaryWriter.ToString().Trim(); + + output.Should().Contain("Test1"); + output.Should().Contain("Test2"); + output.Should().Contain("Test3"); + output.Should().Contain("Test4"); + + _testOutput.WriteLine(output); } [Fact] @@ -65,6 +179,16 @@ public void Test_summary_contains_names_of_failed_tests() .SetFullyQualifiedName("TestProject.SomeTests.Test3") .SetOutcome(TestOutcome.Failed) .SetErrorMessage("ErrorMessage3") + .Build(), + new TestResultBuilder() + .SetDisplayName("Test4") + .SetFullyQualifiedName("TestProject.SomeTests.Test4") + .SetOutcome(TestOutcome.Passed) + .Build(), + new TestResultBuilder() + .SetDisplayName("Test5") + .SetFullyQualifiedName("TestProject.SomeTests.Test5") + .SetOutcome(TestOutcome.Skipped) .Build() ); @@ -74,6 +198,10 @@ public void Test_summary_contains_names_of_failed_tests() output.Should().Contain("Test1"); output.Should().Contain("Test2"); output.Should().Contain("Test3"); + output.Should().NotContain("Test4"); + output.Should().NotContain("Test5"); + + _testOutput.WriteLine(output); } [Fact] @@ -109,6 +237,16 @@ public void Test_summary_contains_error_messages_of_failed_tests() .SetFullyQualifiedName("TestProject.SomeTests.Test3") .SetOutcome(TestOutcome.Failed) .SetErrorMessage("ErrorMessage3") + .Build(), + new TestResultBuilder() + .SetDisplayName("Test4") + .SetFullyQualifiedName("TestProject.SomeTests.Test4") + .SetOutcome(TestOutcome.Passed) + .Build(), + new TestResultBuilder() + .SetDisplayName("Test5") + .SetFullyQualifiedName("TestProject.SomeTests.Test5") + .SetOutcome(TestOutcome.Skipped) .Build() ); @@ -118,5 +256,7 @@ public void Test_summary_contains_error_messages_of_failed_tests() output.Should().Contain("ErrorMessage1"); output.Should().Contain("ErrorMessage2"); output.Should().Contain("ErrorMessage3"); + + _testOutput.WriteLine(output); } } \ No newline at end of file diff --git a/GitHubActionsTestLogger/TestLoggerContext.cs b/GitHubActionsTestLogger/TestLoggerContext.cs index 7cf1dc3..edbd737 100644 --- a/GitHubActionsTestLogger/TestLoggerContext.cs +++ b/GitHubActionsTestLogger/TestLoggerContext.cs @@ -25,7 +25,7 @@ public TestLoggerContext(GitHubWorkflow github, TestLoggerOptions options) Options = options; } - private string ApplyAnnotationFormat(string format, TestResult testResult) + private string FormatAnnotation(string format, TestResult testResult) { var buffer = new StringBuilder(format); @@ -35,7 +35,7 @@ private string ApplyAnnotationFormat(string format, TestResult testResult) // Name token buffer.Replace("$test", testResult.TestCase.DisplayName ?? ""); - // Traits tokens + // Trait tokens foreach (var trait in testResult.Traits.Union(testResult.TestCase.Traits)) buffer.Replace($"$traits.{trait.Name}", trait.Value); @@ -51,11 +51,11 @@ private string ApplyAnnotationFormat(string format, TestResult testResult) return buffer.Trim().ToString(); } - private string ApplyAnnotationTitleFormat(TestResult testResult) => - ApplyAnnotationFormat(Options.AnnotationTitleFormat, testResult); + private string FormatAnnotationTitle(TestResult testResult) => + FormatAnnotation(Options.AnnotationTitleFormat, testResult); - private string ApplyAnnotationMessageFormat(TestResult testResult) => - ApplyAnnotationFormat(Options.AnnotationMessageFormat, testResult); + private string FormatAnnotationMessage(TestResult testResult) => + FormatAnnotation(Options.AnnotationMessageFormat, testResult); public void HandleTestRunStart(TestRunStartEventArgs args) { @@ -73,8 +73,8 @@ public void HandleTestResult(TestResultEventArgs args) if (args.Result.Outcome == TestOutcome.Failed) { _github.CreateErrorAnnotation( - ApplyAnnotationTitleFormat(args.Result), - ApplyAnnotationMessageFormat(args.Result), + FormatAnnotationTitle(args.Result), + FormatAnnotationMessage(args.Result), args.Result.TryGetSourceFilePath(), args.Result.TryGetSourceLine() ); @@ -103,19 +103,34 @@ public void HandleTestRunComplete(TestRunCompleteEventArgs args) "Unknown Target Framework"; var testRunStatistics = new TestRunStatistics( - args.TestRunStatistics?[TestOutcome.Passed] ?? 0, - args.TestRunStatistics?[TestOutcome.Failed] ?? 0, - args.TestRunStatistics?[TestOutcome.Skipped] ?? 0, - args.TestRunStatistics?.ExecutedTests ?? 0, + args.TestRunStatistics?[TestOutcome.Passed] ?? + _testResults.Count(r => r.Outcome == TestOutcome.Passed), + + args.TestRunStatistics?[TestOutcome.Failed] ?? + _testResults.Count(r => r.Outcome == TestOutcome.Failed), + + args.TestRunStatistics?[TestOutcome.Skipped] ?? + _testResults.Count(r => r.Outcome == TestOutcome.Skipped), + + args.TestRunStatistics?.ExecutedTests ?? _testResults.Count, + args.ElapsedTimeInRunningTests ); + var testResults = _testResults + .Where(r => + r.Outcome == TestOutcome.Failed || + r.Outcome == TestOutcome.Passed && Options.SummaryIncludePassedTests || + r.Outcome == TestOutcome.Skipped && Options.SummaryIncludeSkippedTests + ) + .ToArray(); + _github.CreateSummary( TestSummary.Generate( testSuiteName, targetFrameworkName, testRunStatistics, - _testResults + testResults ) ); } diff --git a/GitHubActionsTestLogger/TestLoggerOptions.cs b/GitHubActionsTestLogger/TestLoggerOptions.cs index a2fd4a3..69c88ed 100644 --- a/GitHubActionsTestLogger/TestLoggerOptions.cs +++ b/GitHubActionsTestLogger/TestLoggerOptions.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using GitHubActionsTestLogger.Utils.Extensions; namespace GitHubActionsTestLogger; @@ -7,6 +8,10 @@ public partial class TestLoggerOptions public string AnnotationTitleFormat { get; init; } = "$test"; public string AnnotationMessageFormat { get; init; } = "$error"; + + public bool SummaryIncludePassedTests { get; init; } + + public bool SummaryIncludeSkippedTests { get; init; } } public partial class TestLoggerOptions @@ -21,6 +26,14 @@ public partial class TestLoggerOptions AnnotationMessageFormat = parameters.GetValueOrDefault("annotations.messageFormat") ?? - Default.AnnotationMessageFormat + Default.AnnotationMessageFormat, + + SummaryIncludePassedTests = + parameters.GetValueOrDefault("summary.includePassedTests")?.Pipe(bool.Parse) ?? + Default.SummaryIncludePassedTests, + + SummaryIncludeSkippedTests = + parameters.GetValueOrDefault("summary.includeSkippedTests")?.Pipe(bool.Parse) ?? + Default.SummaryIncludeSkippedTests, }; } \ No newline at end of file diff --git a/GitHubActionsTestLogger/TestRunStatistics.cs b/GitHubActionsTestLogger/TestRunStatistics.cs index d3e10fd..b24aeb8 100644 --- a/GitHubActionsTestLogger/TestRunStatistics.cs +++ b/GitHubActionsTestLogger/TestRunStatistics.cs @@ -1,4 +1,5 @@ using System; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; namespace GitHubActionsTestLogger; @@ -8,4 +9,22 @@ internal record TestRunStatistics( long SkippedTestCount, long TotalTestCount, TimeSpan ElapsedDuration -); \ No newline at end of file +) +{ + public TestOutcome OverallOutcome + { + get + { + if (FailedTestCount > 0) + return TestOutcome.Failed; + + if (PassedTestCount > 0) + return TestOutcome.Passed; + + if (SkippedTestCount > 0) + return TestOutcome.Skipped; + + return TestOutcome.None; + } + } +} \ No newline at end of file diff --git a/GitHubActionsTestLogger/TestSummary.cs b/GitHubActionsTestLogger/TestSummary.cs index 32ebed5..b3c5e08 100644 --- a/GitHubActionsTestLogger/TestSummary.cs +++ b/GitHubActionsTestLogger/TestSummary.cs @@ -20,7 +20,12 @@ public static string Generate( buffer .Append("
") .Append("") - .Append(testRunStatistics.FailedTestCount <= 0 ? "🟢" : "🔴") + .Append(testRunStatistics.OverallOutcome switch + { + TestOutcome.Passed => "🟢", + TestOutcome.Failed => "🔴", + _ => "🟡" + }) .Append(" ") .Append("") .Append(testSuiteName) @@ -100,35 +105,62 @@ public static string Generate( .AppendLine() .AppendLine(); - // List of failed tests - foreach (var testResult in testResults.Where(r => r.Outcome == TestOutcome.Failed)) + // Test results + var testResultsOrdered = testResults + .OrderByDescending(r => r.Outcome == TestOutcome.Failed) + .ThenByDescending(r => r.Outcome == TestOutcome.Passed); + + foreach (var testResult in testResultsOrdered) { - // Generate permalink for the test + // Generate permalink for the test source var filePath = testResult.TryGetSourceFilePath(); var fileLine = testResult.TryGetSourceLine(); - var url = !string.IsNullOrWhiteSpace(filePath) - ? GitHubWorkflow.TryGenerateFilePermalink(filePath, fileLine) - : null; + var url = filePath?.Pipe(p => GitHubWorkflow.TryGenerateFilePermalink(p, fileLine)); + + buffer + .Append("- ") + .Append(testResult.Outcome switch + { + TestOutcome.Passed => "🟢", + TestOutcome.Failed => "🔴", + _ => "🟡" + }) + .Append(" "); + + if (!string.IsNullOrWhiteSpace(url)) + { + buffer + .Append("["); + } buffer - .Append("- Fail: ") - .Append("[") .Append("**") .Append(testResult.TestCase.DisplayName) - .Append("**") - .Append("]") - .Append("(") - .Append(url ?? "#") - .Append(")") - .AppendLine() + .Append("**"); + + if (!string.IsNullOrWhiteSpace(url)) + { + buffer + .Append("]") + .Append("(") + .Append(url) + .Append(")"); + } + + buffer.AppendLine(); + + if (!string.IsNullOrWhiteSpace(testResult.ErrorMessage)) + { // YAML syntax highlighting works really well for exception messages and stack traces - .AppendLine("```yml") - .AppendLine(testResult.ErrorMessage) - .AppendLine(testResult.ErrorStackTrace) - .AppendLine("```"); + buffer + .Append(" ").AppendLine("```yml") + .Append(" ").AppendLine(testResult.ErrorMessage) + .Append(" ").AppendLine(testResult.ErrorStackTrace) + .Append(" ").AppendLine("```"); + } } - // Spoiler closing tags + // Spoiler closing tag buffer .Append("
") .AppendLine() diff --git a/Readme.md b/Readme.md index 9db70ee..085ea9a 100644 --- a/Readme.md +++ b/Readme.md @@ -39,8 +39,8 @@ To learn more about the war and how you can help, [click here](https://tyrrrz.me To use **GitHub Actions Test Logger**, follow these steps: -1. Install **GitHubActionsTestLogger** package in your test project -2. Update **Microsoft.NET.Test.Sdk** package in your test project to the latest version +1. Install the **GitHubActionsTestLogger** package in your test project +2. Update the **Microsoft.NET.Test.Sdk** package in your test project to the latest version 3. Modify your GitHub Actions workflow file by adding `--logger GitHubActions` to `dotnet test`: ```yaml @@ -111,3 +111,23 @@ Supports the same replacement tokens as [`annotations.titleFormat`](#custom-anno - `$error` → `AssertionException: Expected 'true' but found 'false'` - `$error\n$trace` → `AssertionException: Expected 'true' but found 'false'`, followed by stacktrace on the next line + +#### Include passed tests in summary + +Use the `summary.includePassedTests` option to specify whether passed tests should be included in the summary. + +**Default**: `false`. + +> **Warning**: +> GitHub job summaries are limited to `1MB` worth of content for each workflow step. +> If your project has a large number of tests, you may want to keep this option disabled to avoid hitting the limit. + +#### Include skipped tests in summary + +Use the `summary.includeSkippedTests` option to specify whether skipped tests should be included in the summary. + +**Default**: `false`. + +> **Warning**: +> GitHub job summaries are limited to `1MB` worth of content for each workflow step. +> If your project has a large number of tests, you may want to keep this option disabled to avoid hitting the limit. \ No newline at end of file