Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for code coverage using Coverlet + ReportGenerator #3919

Closed
wants to merge 1 commit into from
Closed

Add support for code coverage using Coverlet + ReportGenerator #3919

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src/Microsoft.DotNet.Arcade.Sdk/tools/Coverlet/Coverlet.props
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
<Project>
<PropertyGroup>
<CoverletFormat Condition="'$(CoverletFormat)' == ''">opencover</CoverletFormat>
<CoverletThreshold Condition="$(CoverletThreshold) == ''">0</CoverletThreshold>
<CoverletThresholdType Condition="$(CoverletThresholdType) == ''">line,branch,method</CoverletThresholdType>
<CoverletThresholdStat Condition="$(CoverletThresholdStat) == ''">minimum</CoverletThresholdStat>
<CoverletVerbosity Condition="$(CoverletVerbosity) == ''">normal</CoverletVerbosity>

<ReportGeneratorReportType Condition="$(ReportGeneratorReportType) == ''">Html</ReportGeneratorReportType>
<ReportGeneratorVerbosity Condition="$(ReportGeneratorVerbosity) == ''">warning</ReportGeneratorVerbosity>
</PropertyGroup>
</Project>
35 changes: 35 additions & 0 deletions src/Microsoft.DotNet.Arcade.Sdk/tools/Coverlet/Coverlet.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
<Project>

<Target Name="RunCodeCoverage"
Inputs="@(TestRunWithRunner)"
Outputs="%(TestRunWithRunner.ResultsCodeCoveragePath)">

<Telemetry EventName="NETCORE_ENGINEERING_TELEMETRY" EventData="Category=CodeCoverage" />

<PropertyGroup>
<_TestAssembly>%(TestRunWithRunner.Identity)</_TestAssembly>
<_CoverageOutputFile>%(TestRunWithRunner.ResultsCodeCoveragePath)</_CoverageOutputFile>
<_TestRunner>%(TestRunWithRunner.TestRunner)</_TestRunner>
<_TestRunner>$(_TestRunner.Replace('"', '\"'))</_TestRunner>
<_TestRunnerArgs>%(TestRunWithRunner.TestRunnerArgs)</_TestRunnerArgs>
<_TestRunnerArgs>$(_TestRunnerArgs.Replace('"', '\"'))</_TestRunnerArgs>
<_ReportCoverageDirectory>$(ArtifactsLogDir)codecoverage/$([System.IO.Path]::GetFileNameWithoutExtension('$(_TestAssembly)'))</_ReportCoverageDirectory>

<_CoverletArgs>"$(_TestAssembly)" --format $(CoverletFormat) --output "$(_CoverageOutputFile)" --threshold $(CoverletThreshold) --threshold-type "$(CoverletThresholdType)" --threshold-stat $(CoverletThresholdStat) --verbosity $(CoverletVerbosity) --target "$(_TestRunner)" --targetargs "$(_TestRunnerArgs)"</_CoverletArgs>
<_CoverletCommand>"$(DotNetTool)" tool run coverlet $(_CoverletArgs)</_CoverletCommand>
</PropertyGroup>

<Exec Command="$(_CoverletCommand)" Condition="'%(TestRunWithRunner.TestRuntime)' == 'Core'" WorkingDirectory="$(_TargetDir)" />

<Message Text="Coverlet does not support non-Core projects" Condition="'%(TestRunWithRunner.TestRuntime)' != 'Core'" Importance="Low" />

<PropertyGroup>
<_ReportGeneratorArgs>"--reports:$(_CoverageOutputFile)" "--targetdir:$(_ReportCoverageDirectory)" "-reporttypes:$(ReportGeneratorReportType)" "-verbosity:$(ReportGeneratorVerbosity)"</_ReportGeneratorArgs>
<_ReportGeneratorCommand>"$(DotNetTool)" tool run reportgenerator $(_ReportGeneratorArgs)</_ReportGeneratorCommand>
</PropertyGroup>

<Exec Command="$(_ReportGeneratorCommand)" Condition="Exists('$(_CoverageOutputFile)')" />
</Target>
</Project>
8 changes: 7 additions & 1 deletion src/Microsoft.DotNet.Arcade.Sdk/tools/Tests.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<IsPerformanceTestProject>false</IsPerformanceTestProject>
<IsPerformanceTestProject Condition="$(MSBuildProjectName.EndsWith('.PerformanceTests'))">true</IsPerformanceTestProject>
</PropertyGroup>

<PropertyGroup Condition="'$(IsIntegrationTestProject)' == ''">
<IsIntegrationTestProject>false</IsIntegrationTestProject>
<IsIntegrationTestProject Condition="$(MSBuildProjectName.EndsWith('.IntegrationTests'))">true</IsIntegrationTestProject>
Expand Down Expand Up @@ -39,10 +39,16 @@
<!-- Default test runner -->
<TestRunnerName Condition="'$(UsingToolXUnit)' == 'true'">XUnit</TestRunnerName>

<!-- Default code coverage -->
<CodeCoverageRunnerName>Coverlet</CodeCoverageRunnerName>

<!-- exclude test projects from source-build by default -->
<ExcludeFromSourceBuild Condition="'$(ExcludeFromSourceBuild)' == ''">true</ExcludeFromSourceBuild>
</PropertyGroup>

<!-- Import specialized props files of supported test runners -->
<Import Project="$(MSBuildThisFileDirectory)$(TestRunnerName)\$(TestRunnerName).props" Condition="'$(TestRunnerName)' != '' and Exists('$(MSBuildThisFileDirectory)$(TestRunnerName)\$(TestRunnerName).props')"/>

<!-- Import specialized props files of supported code coverage tool -->
<Import Project="$(MSBuildThisFileDirectory)$(CodeCoverageRunnerName)\$(CodeCoverageRunnerName).props" Condition="'$(CodeCoverageRunnerName)' != '' and Exists('$(MSBuildThisFileDirectory)$(CodeCoverageRunnerName)\$(CodeCoverageRunnerName).props')"/>
</Project>
21 changes: 20 additions & 1 deletion src/Microsoft.DotNet.Arcade.Sdk/tools/Tests.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,27 @@
<TestRuntime Condition="'$(TestRuntime)' == '' and '$(OS)' == 'Windows_NT'">Full</TestRuntime>
</PropertyGroup>

<PropertyGroup Condition="'$(RunCodeCoverage)' == 'true'">
<DeterministicSourcePaths>false</DeterministicSourcePaths>
</PropertyGroup>

<PropertyGroup Condition="'$(IsTestProject)' == 'true' and '$(TestArchitectures)' == ''">
<TestArchitectures>$(PlatformTarget)</TestArchitectures>
<TestArchitectures Condition="'$(PlatformTarget)' == '' or '$(PlatformTarget)' == 'AnyCpu'">x64</TestArchitectures>

<TestDependsOn Condition="'$(RunCodeCoverage)' != 'true'">
$(_GetTestsToRunTarget);
RunTests
</TestDependsOn>

<TestDependsOn Condition="'$(RunCodeCoverage)' == 'true'">
$(_GetTestsToRunTarget);
PrepareForRunTests;
RunCodeCoverage;
</TestDependsOn>
</PropertyGroup>

<Target Name="Test" DependsOnTargets="$(_GetTestsToRunTarget);RunTests" Condition="'$(IsUnitTestProject)' == 'true' or '$(IsPerformanceTestProject)' == 'true'" />
<Target Name="Test" DependsOnTargets="$(TestDependsOn)" Condition="'$(IsUnitTestProject)' == 'true' or '$(IsPerformanceTestProject)' == 'true'" />
<Target Name="IntegrationTest" DependsOnTargets="$(_GetTestsToRunTarget);RunTests" Condition="'$(IsIntegrationTestProject)' == 'true'" />

<ItemGroup>
Expand Down Expand Up @@ -50,6 +65,7 @@
<ResultsXmlPath>$(ArtifactsTestResultsDir)$(_ResultFileNameNoExt).xml</ResultsXmlPath>
<ResultsHtmlPath>$(ArtifactsTestResultsDir)$(_ResultFileNameNoExt).html</ResultsHtmlPath>
<ResultsStdOutPath>$(ArtifactsLogDir)$(_ResultFileNameNoExt).log</ResultsStdOutPath>
<ResultsCodeCoveragePath>$(ArtifactsLogDir)$(_ResultFileNameNoExt).codecoverage.xml</ResultsCodeCoveragePath>
<TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
</TestToRun>
</ItemGroup>
Expand All @@ -69,4 +85,7 @@

<!-- Import specialized targets files of supported test runners -->
<Import Project="$(MSBuildThisFileDirectory)$(TestRunnerName)\$(TestRunnerName).targets" Condition="'$(TestRunnerName)' != '' and Exists('$(MSBuildThisFileDirectory)$(TestRunnerName)\$(TestRunnerName).targets')"/>

<!-- Import specialized targets files of supported code coverage runners -->
<Import Project="$(MSBuildThisFileDirectory)$(CodeCoverageRunnerName)\$(CodeCoverageRunnerName).targets" Condition="'$(CodeCoverageRunnerName)' != '' and Exists('$(MSBuildThisFileDirectory)$(CodeCoverageRunnerName)\$(CodeCoverageRunnerName).targets')"/>
</Project>
62 changes: 41 additions & 21 deletions src/Microsoft.DotNet.Arcade.Sdk/tools/XUnit/XUnit.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,11 @@
Condition="'$(XUnitCoreSettingsFile)' != '' and '$(TargetFrameworkIdentifier)' == '.NETCoreApp'" />
</ItemGroup>

<!--
Include '*' target to force running tests even if the input assemblies haven't changed and the outputs are present.
This matches the common expectations that test command always runs all tests in scope.
-->
<Target Name="RunTests"
Inputs="@(TestToRun);*"
Outputs="%(TestToRun.ResultsStdOutPath);%(TestToRun.ResultsXmlPath);%(TestToRun.ResultsHtmlPath)">
<Telemetry EventName="NETCORE_ENGINEERING_TELEMETRY" EventData="Category=Test" />

<Target Name="PrepareForRunTests"
Inputs="%(TestToRun.Identity)"
Outputs="%(TestToRun.Identity);*">

<PropertyGroup>
<_TestEnvironment>%(TestToRun.EnvironmentDisplay)</_TestEnvironment>
<_TestAssembly>%(TestToRun.Identity)</_TestAssembly>
Expand All @@ -47,10 +44,10 @@
<_TargetDir>$([System.IO.Path]::GetDirectoryName('$(_TestAssembly)'))\</_TargetDir>
<_CoreRuntimeConfigPath>$(_TargetDir)$(_TargetFileNameNoExt).runtimeconfig.json</_CoreRuntimeConfigPath>
<_CoreDepsPath>$(_TargetDir)$(_TargetFileNameNoExt).deps.json</_CoreDepsPath>

<_TestRunner Condition="'%(TestToRun.Architecture)'=='x86' And Exists('$(DotNetRoot)x86\dotnet.exe')">$(DotNetRoot)x86\dotnet.exe</_TestRunner>
<_TestRunner Condition="'$(_TestRunner)'==''">$(DotNetTool)</_TestRunner>

<_TestRunnerArgs>exec --depsfile "$(_CoreDepsPath)" --runtimeconfig "$(_CoreRuntimeConfigPath)" $(TestRuntimeAdditionalArguments) "$(NuGetPackageRoot)xunit.runner.console/$(XUnitVersion)/tools/$(_TestRunnerTargetFramework)/xunit.console.dll" "$(_TestAssembly)" -noautoreporters -xml "%(TestToRun.ResultsXmlPath)" -html "%(TestToRun.ResultsHtmlPath)" $(_TestRunnerAdditionalArguments)</_TestRunnerArgs>
</PropertyGroup>

Expand All @@ -66,20 +63,43 @@
<_TestRunner Condition="'$(_TestRuntime)' != 'Mono'">$(_XUnitConsoleExePath)</_TestRunner>
</PropertyGroup>

<ItemGroup>
<TestRunWithRunner Include="@(TestToRun)">
<TestRunner>$(_TestRunner)</TestRunner>
<TestRunnerArgs>$(_TestRunnerArgs)</TestRunnerArgs>
</TestRunWithRunner>
</ItemGroup>

</Target>

<!--
Include '*' target to force running tests even if the input assemblies haven't changed and the outputs are present.
This matches the common expectations that test command always runs all tests in scope.
-->
<Target Name="RunTests"
DependsOnTargets="PrepareForRunTests"
Inputs="@(TestRunWithRunner);*"
Outputs="%(TestRunWithRunner.ResultsStdOutPath);%(TestRunWithRunner.ResultsXmlPath);%(TestRunWithRunner.ResultsHtmlPath)">
<Telemetry EventName="NETCORE_ENGINEERING_TELEMETRY" EventData="Category=Test" />

<PropertyGroup>
<_TestRunnerCommand>"$(_TestRunner)" $(_TestRunnerArgs)</_TestRunnerCommand>
<_TestEnvironment>%(TestRunWithRunner.EnvironmentDisplay)</_TestEnvironment>
<_TestAssembly>%(TestRunWithRunner.Identity)</_TestAssembly>
<_TestRunnerAdditionalArguments>%(TestRunWithRunner.TestRunnerAdditionalArguments)</_TestRunnerAdditionalArguments>

<!--
<_TestRunnerCommand>"%(TestRunWithRunner.TestRunner)" %(TestRunWithRunner.TestRunnerArgs)</_TestRunnerCommand>

<!--
Redirect std output of the runner.
Note that xUnit outputs failure info to both STDOUT (stack trace, message) and STDERR (failed test name)
Note that xUnit outputs failure info to both STDOUT (stack trace, message) and STDERR (failed test name)
-->
<_TestRunnerCommand Condition="'$(TestCaptureOutput)' != 'false'">$(_TestRunnerCommand) > "%(TestToRun.ResultsStdOutPath)" 2>&amp;1</_TestRunnerCommand>
<_TestRunnerCommand Condition="'$(TestCaptureOutput)' != 'false'">$(_TestRunnerCommand) > "%(TestRunWithRunner.ResultsStdOutPath)" 2>&amp;1</_TestRunnerCommand>
</PropertyGroup>

<ItemGroup>
<_OutputFiles Include="%(TestToRun.ResultsXmlPath)" />
<_OutputFiles Include="%(TestToRun.ResultsHtmlPath)" />
<_OutputFiles Include="%(TestToRun.ResultsStdOutPath)" />
<_OutputFiles Include="%(TestRunWithRunner.ResultsXmlPath)" />
<_OutputFiles Include="%(TestRunWithRunner.ResultsHtmlPath)" />
<_OutputFiles Include="%(TestRunWithRunner.ResultsStdOutPath)" />
</ItemGroup>

<MakeDir Directories="@(_OutputFiles->'%(RootDir)%(Directory)')"/>
Expand All @@ -93,8 +113,8 @@
<!--
Add command line to the log.
-->
<WriteLinesToFile File="%(TestToRun.ResultsStdOutPath)"
Overwrite="false"
<WriteLinesToFile File="%(TestRunWithRunner.ResultsStdOutPath)"
Overwrite="false"
Lines=";=== COMMAND LINE ===;$(_TestRunnerCommand)"
Condition="'$(TestCaptureOutput)' != 'false'" />

Expand All @@ -104,11 +124,11 @@
<Message Text="Tests succeeded: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" />

<PropertyGroup>
<_ResultsFileToDisplay>%(TestToRun.ResultsHtmlPath)</_ResultsFileToDisplay>
<_ResultsFileToDisplay>%(TestRunWithRunner.ResultsHtmlPath)</_ResultsFileToDisplay>
<_ResultsFileToDisplay Condition="!Exists('$(_ResultsFileToDisplay)')">%(TestToRun.ResultsStdOutPath)</_ResultsFileToDisplay>
</PropertyGroup>

<!--
<!--
Ideally we would set ContinueOnError="ErrorAndContinue" so that when a test fails in multi-targeted test project
we'll still run tests for all target frameworks. ErrorAndContinue doesn't work well on Linux though: https://github.com/Microsoft/msbuild/issues/3961.
-->
Expand Down