From d8ff02ddc8cc4c081d113d337e04dd7931b49223 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Fri, 15 Mar 2024 18:01:23 +0100 Subject: [PATCH 1/3] Support "Central Package Managemen" with the `MSTest.Sdk` --- .../Sdk/Runner/ClassicEngine.targets | 20 ++-- .../MSTest.Sdk/Sdk/Runner/NativeAOT.targets | 14 +-- .../MSTest.Sdk/Sdk/VSTest/VSTest.targets | 2 +- .../SdkTests.cs | 98 +++++++++++++++++++ .../DotnetCli.cs | 47 ++++++--- 5 files changed, 149 insertions(+), 32 deletions(-) diff --git a/src/Package/MSTest.Sdk/Sdk/Runner/ClassicEngine.targets b/src/Package/MSTest.Sdk/Sdk/Runner/ClassicEngine.targets index 1d3b162fdf..c4b9f224e3 100644 --- a/src/Package/MSTest.Sdk/Sdk/Runner/ClassicEngine.targets +++ b/src/Package/MSTest.Sdk/Sdk/Runner/ClassicEngine.targets @@ -28,20 +28,20 @@ - - - - + + + + - - - - - - + + + + + + diff --git a/src/Package/MSTest.Sdk/Sdk/Runner/NativeAOT.targets b/src/Package/MSTest.Sdk/Sdk/Runner/NativeAOT.targets index 81eee6d70c..5ebaae4aea 100644 --- a/src/Package/MSTest.Sdk/Sdk/Runner/NativeAOT.targets +++ b/src/Package/MSTest.Sdk/Sdk/Runner/NativeAOT.targets @@ -4,18 +4,18 @@ - - - - + + + + - - + + - + diff --git a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets index 416bf42fca..85d2afff90 100644 --- a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets +++ b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets @@ -2,7 +2,7 @@ - + diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index ceece67d63..93b825cd31 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -35,6 +35,14 @@ public async Task RunTests_With_VSTest(string multiTfm, BuildConfiguration build .PatchCodeWithReplace("$Extensions$", string.Empty), addPublicFeeds: true); + File.WriteAllText(Path.Combine(generator.TargetAssetPath, "Directory.Packages.props"), """ + + + true + + +"""); + var compilationResult = await DotnetCli.RunAsync($"test -c {buildConfiguration} {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); Assert.AreEqual(0, compilationResult.ExitCode); @@ -63,6 +71,14 @@ public async Task RunTests_With_MSTestRunner_DotnetTest(string multiTfm, BuildCo .PatchCodeWithReplace("$Extensions$", string.Empty), addPublicFeeds: true); + File.WriteAllText(Path.Combine(generator.TargetAssetPath, "Directory.Packages.props"), """ + + + true + + +"""); + var compilationResult = await DotnetCli.RunAsync($"test -c {buildConfiguration} {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); Assert.AreEqual(0, compilationResult.ExitCode); @@ -91,6 +107,47 @@ public async Task RunTests_With_MSTestRunner_Standalone(string multiTfm, BuildCo .PatchCodeWithReplace("$Extensions$", string.Empty), addPublicFeeds: true); + File.WriteAllText(Path.Combine(generator.TargetAssetPath, "Directory.Packages.props"), """ + + + true + + +"""); + + var compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); + Assert.AreEqual(0, compilationResult.ExitCode); + foreach (var tfm in multiTfm.Split(";")) + { + var testHost = TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); + var testHostResult = await testHost.ExecuteAsync(); + testHostResult.AssertOutputContains("Passed! - Failed: 0, Passed: 1, Skipped: 0, Total: 1"); + } + } + + [ArgumentsProvider(nameof(GetBuildMatrixMultiTfmFoldedBuildConfiguration))] + public async Task RunTests_With_CentralPackageManagement_Standalone(string multiTfm, BuildConfiguration buildConfiguration) + { + using TestAsset generator = await TestAsset.GenerateAssetAsync( + AssetName, + SourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$OutputType$", string.Empty) + .PatchCodeWithReplace("$TargetFramework$", $"{multiTfm}") + .PatchCodeWithReplace("$EnableMSTestRunner$", string.Empty) + .PatchCodeWithReplace("$TestingPlatformDotnetTestSupport$", string.Empty) + .PatchCodeWithReplace("$ExtraProperties$", string.Empty) + .PatchCodeWithReplace("$Extensions$", string.Empty), + addPublicFeeds: true); + + File.WriteAllText(Path.Combine(generator.TargetAssetPath, "Directory.Packages.props"), """ + + + true + + +"""); + var compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); Assert.AreEqual(0, compilationResult.ExitCode); foreach (var tfm in multiTfm.Split(";")) @@ -160,6 +217,14 @@ public async Task RunTests_With_MSTestRunner_Standalone_Selectively_Enabled_Exte .PatchCodeWithReplace("$Extensions$", msbuildExtensionEnableFragment), addPublicFeeds: true); + File.WriteAllText(Path.Combine(generator.TargetAssetPath, "Directory.Packages.props"), """ + + + true + + +"""); + var compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); Assert.AreEqual(0, compilationResult.ExitCode); foreach (var tfm in multiTfm.Split(";")) @@ -188,6 +253,14 @@ public async Task RunTests_With_MSTestRunner_Standalone_EnableAll_Extensions(str .PatchCodeWithReplace("$Extensions$", "AllMicrosoft"), addPublicFeeds: true); + File.WriteAllText(Path.Combine(generator.TargetAssetPath, "Directory.Packages.props"), """ + + + true + + +"""); + var compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); Assert.AreEqual(0, compilationResult.ExitCode); foreach (var tfm in multiTfm.Split(";")) @@ -227,6 +300,14 @@ public async Task RunTests_With_MSTestRunner_Standalone_Enable_Default_Extension .PatchCodeWithReplace("$Extensions$", enableDefaultExtensions ? string.Empty : "None"), addPublicFeeds: true); + File.WriteAllText(Path.Combine(generator.TargetAssetPath, "Directory.Packages.props"), """ + + + true + + +"""); + var compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); Assert.AreEqual(0, compilationResult.ExitCode); foreach (var tfm in multiTfm.Split(";")) @@ -259,6 +340,14 @@ public async Task Invalid_TestingProfile_Name_Should_Fail(string multiTfm, Build .PatchCodeWithReplace("$Extensions$", "WrongName"), addPublicFeeds: true); + File.WriteAllText(Path.Combine(generator.TargetAssetPath, "Directory.Packages.props"), """ + + + true + + +"""); + var compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path, failIfReturnValueIsNotZero: false); Assert.AreEqual(1, compilationResult.ExitCode); compilationResult.AssertOutputContains("Invalid value for property TestingExtensionsProfile. Valid values are 'Default', 'AllMicrosoft' and 'None'."); @@ -287,10 +376,19 @@ await RetryHelper.RetryAsync( .PatchCodeWithReplace("$ExtraProperties$", """ true false + $(NoWarn);NU1507 """) .PatchCodeWithReplace("$Extensions$", string.Empty), addPublicFeeds: true); + File.WriteAllText(Path.Combine(generator.TargetAssetPath, "Directory.Packages.props"), """ + + + true + + +"""); + var compilationResult = await DotnetCli.RunAsync($"publish -r {RID} {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); compilationResult.AssertOutputNotContains("warning"); compilationResult.AssertOutputContains("Generating native code"); diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs index 9f542a9fee..639176d75d 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs @@ -47,6 +47,8 @@ public static int MaxOutstandingCommands } } + public static bool DoNotRetry { get; set; } + public static async Task RunAsync( string args, string nugetGlobalPackagesFolder, @@ -88,24 +90,41 @@ public static async Task RunAsync( environmentVariables["NUGET_PACKAGES"] = nugetGlobalPackagesFolder; - var delay = Backoff.ExponentialBackoff(TimeSpan.FromSeconds(3), retryCount: 5, factor: 1.5); - return await Policy - .Handle() - .WaitAndRetryAsync(delay) - .ExecuteAsync(async () => + if (DoNotRetry) + { + if (args.StartsWith("dotnet ", StringComparison.OrdinalIgnoreCase)) { - if (args.StartsWith("dotnet ", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException("Command should not start with 'dotnet'"); + } + + using var dotnet = new DotnetMuxer(environmentVariables); + int exitCode = await dotnet.Args(args, workingDirectory, timeoutInSeconds); + + return exitCode != 0 && failIfReturnValueIsNotZero + ? throw new InvalidOperationException($"Command 'dotnet {args}' failed.\n\nStandardOutput:\n{dotnet.StandardOutput}\nStandardError:\n{dotnet.StandardError}") + : new DotnetMuxerResult(args, exitCode, dotnet.StandardOutput, dotnet.StandardOutputLines, dotnet.StandardError, dotnet.StandardErrorLines); + } + else + { + var delay = Backoff.ExponentialBackoff(TimeSpan.FromSeconds(3), retryCount: 5, factor: 1.5); + return await Policy + .Handle() + .WaitAndRetryAsync(delay) + .ExecuteAsync(async () => { - throw new InvalidOperationException("Command should not start with 'dotnet'"); - } + if (args.StartsWith("dotnet ", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Command should not start with 'dotnet'"); + } - using var dotnet = new DotnetMuxer(environmentVariables); - int exitCode = await dotnet.Args(args, workingDirectory, timeoutInSeconds); + using var dotnet = new DotnetMuxer(environmentVariables); + int exitCode = await dotnet.Args(args, workingDirectory, timeoutInSeconds); - return exitCode != 0 && failIfReturnValueIsNotZero - ? throw new InvalidOperationException($"Command 'dotnet {args}' failed.\n\nStandardOutput:\n{dotnet.StandardOutput}\nStandardError:\n{dotnet.StandardError}") - : new DotnetMuxerResult(args, exitCode, dotnet.StandardOutput, dotnet.StandardOutputLines, dotnet.StandardError, dotnet.StandardErrorLines); - }); + return exitCode != 0 && failIfReturnValueIsNotZero + ? throw new InvalidOperationException($"Command 'dotnet {args}' failed.\n\nStandardOutput:\n{dotnet.StandardOutput}\nStandardError:\n{dotnet.StandardError}") + : new DotnetMuxerResult(args, exitCode, dotnet.StandardOutput, dotnet.StandardOutputLines, dotnet.StandardError, dotnet.StandardErrorLines); + }); + } } finally { From 46b4ff0fb84809f3f2b89d47573f5cef3efa4dcc Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Fri, 15 Mar 2024 18:03:06 +0100 Subject: [PATCH 2/3] No warn CPM --- .../MSTest.Acceptance.IntegrationTests/SdkTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 93b825cd31..8a97ddec7e 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -375,8 +375,7 @@ await RetryHelper.RetryAsync( .PatchCodeWithReplace("$TestingPlatformDotnetTestSupport$", string.Empty) .PatchCodeWithReplace("$ExtraProperties$", """ true - false - $(NoWarn);NU1507 + false """) .PatchCodeWithReplace("$Extensions$", string.Empty), addPublicFeeds: true); @@ -409,6 +408,7 @@ await RetryHelper.RetryAsync( $TestingPlatformDotnetTestSupport$ $ExtraProperties$ x64 + $(NoWarn);NU1507 From 93a9e316d157bbde581009c27c79a4e3093a1bf2 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Fri, 15 Mar 2024 18:16:57 +0100 Subject: [PATCH 3/3] fix the baseline --- .../SdkTests.cs | 2 +- .../testsbaseline.txt | 4 +- .../DotnetCli.cs | 39 ++++++++----------- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 8a97ddec7e..c4a66080d9 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -375,7 +375,7 @@ await RetryHelper.RetryAsync( .PatchCodeWithReplace("$TestingPlatformDotnetTestSupport$", string.Empty) .PatchCodeWithReplace("$ExtraProperties$", """ true - false + false """) .PatchCodeWithReplace("$Extensions$", string.Empty), addPublicFeeds: true); diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/testsbaseline.txt b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/testsbaseline.txt index 927f5d2d4a..a2ef5f1b15 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/testsbaseline.txt +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/testsbaseline.txt @@ -71,6 +71,8 @@ MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.Messages.V MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.Invalid_TestingProfile_Name_Should_Fail(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (multitfm,Debug) MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.Invalid_TestingProfile_Name_Should_Fail(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (multitfm,Release) MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.NativeAot_Smoke_Test_On_Windows() +MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.RunTests_With_CentralPackageManagement_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (multitfm,Debug) +MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.RunTests_With_CentralPackageManagement_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (multitfm,Release) MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.RunTests_With_MSTestRunner_DotnetTest(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (multitfm,Debug) MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.RunTests_With_MSTestRunner_DotnetTest(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (multitfm,Release) MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.RunTests_With_MSTestRunner_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (multitfm,Debug) @@ -93,4 +95,4 @@ MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.R MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.RunTests_With_MSTestRunner_Standalone_Selectively_Enabled_Extensions(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration, string, string, string) (multitfm,Release,TrxReport) MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.RunTests_With_VSTest(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (multitfm,Debug) MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.SdkTests.RunTests_With_VSTest(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (multitfm,Release) -MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.ValueTaskTests.CanUseValueTaskForAllKnownLocations() +MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.ValueTaskTests.CanUseValueTaskForAllKnownLocations() \ No newline at end of file diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs index 639176d75d..45de37809b 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs @@ -92,17 +92,7 @@ public static async Task RunAsync( if (DoNotRetry) { - if (args.StartsWith("dotnet ", StringComparison.OrdinalIgnoreCase)) - { - throw new InvalidOperationException("Command should not start with 'dotnet'"); - } - - using var dotnet = new DotnetMuxer(environmentVariables); - int exitCode = await dotnet.Args(args, workingDirectory, timeoutInSeconds); - - return exitCode != 0 && failIfReturnValueIsNotZero - ? throw new InvalidOperationException($"Command 'dotnet {args}' failed.\n\nStandardOutput:\n{dotnet.StandardOutput}\nStandardError:\n{dotnet.StandardError}") - : new DotnetMuxerResult(args, exitCode, dotnet.StandardOutput, dotnet.StandardOutputLines, dotnet.StandardError, dotnet.StandardErrorLines); + return await CallTheMuxer(args, environmentVariables, workingDirectory, timeoutInSeconds, failIfReturnValueIsNotZero); } else { @@ -112,17 +102,7 @@ public static async Task RunAsync( .WaitAndRetryAsync(delay) .ExecuteAsync(async () => { - if (args.StartsWith("dotnet ", StringComparison.OrdinalIgnoreCase)) - { - throw new InvalidOperationException("Command should not start with 'dotnet'"); - } - - using var dotnet = new DotnetMuxer(environmentVariables); - int exitCode = await dotnet.Args(args, workingDirectory, timeoutInSeconds); - - return exitCode != 0 && failIfReturnValueIsNotZero - ? throw new InvalidOperationException($"Command 'dotnet {args}' failed.\n\nStandardOutput:\n{dotnet.StandardOutput}\nStandardError:\n{dotnet.StandardError}") - : new DotnetMuxerResult(args, exitCode, dotnet.StandardOutput, dotnet.StandardOutputLines, dotnet.StandardError, dotnet.StandardErrorLines); + return await CallTheMuxer(args, environmentVariables, workingDirectory, timeoutInSeconds, failIfReturnValueIsNotZero); }); } } @@ -131,4 +111,19 @@ public static async Task RunAsync( s_maxOutstandingCommands_semaphore.Release(); } } + + private static async Task CallTheMuxer(string args, Dictionary environmentVariables, string? workingDirectory, int timeoutInSeconds, bool failIfReturnValueIsNotZero) + { + if (args.StartsWith("dotnet ", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Command should not start with 'dotnet'"); + } + + using var dotnet = new DotnetMuxer(environmentVariables); + int exitCode = await dotnet.Args(args, workingDirectory, timeoutInSeconds); + + return exitCode != 0 && failIfReturnValueIsNotZero + ? throw new InvalidOperationException($"Command 'dotnet {args}' failed.\n\nStandardOutput:\n{dotnet.StandardOutput}\nStandardError:\n{dotnet.StandardError}") + : new DotnetMuxerResult(args, exitCode, dotnet.StandardOutput, dotnet.StandardOutputLines, dotnet.StandardError, dotnet.StandardErrorLines); + } }