From a3a75bde0599518e68d66324858cb3e9bee76db0 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Wed, 3 Apr 2024 13:48:09 +0000 Subject: [PATCH] Make SmokeTests use repo infrastructure This allows the test project to just be invoked with `dotnet test` or `dotnet build /t:Test` after the VMR is built. 1. Stop depending on environment variables for state instead use RuntimeConfiguration values same as UnifiedBuild test project. Allow facts and theories to be conditioned based on boolean values instead of env vars. 2. Update the README with the new msbuild property switch names 3. Delete dead-code and the checked-in .sln file --- src/SourceBuild/content/.gitignore | 3 - src/SourceBuild/content/Directory.Build.props | 1 + .../content/Directory.Packages.props | 2 + src/SourceBuild/content/build.proj | 31 +++-- src/SourceBuild/content/build.sh | 2 +- src/SourceBuild/content/eng/Versions.props | 10 +- .../content/eng/build.sourcebuild.targets | 72 ----------- .../content/eng/tools/init-build.proj | 8 -- .../ArtifactsSizeTest.cs | 6 +- .../ConditionalFactAttribute.cs | 29 +++++ .../ConditionalTheoryAttribute.cs | 18 +++ .../Config.cs | 57 ++++----- .../Directory.Build.props | 10 -- .../DotNetHelper.cs | 2 +- .../LicenseScanTests.cs | 4 +- ...osoft.DotNet.SourceBuild.SmokeTests.csproj | 116 +++++++++++++++--- ...icrosoft.DotNet.SourceBuild.SmokeTests.sln | 25 ---- .../OmniSharpTests.cs | 4 +- .../PoisonTests.cs | 4 +- .../README.md | 23 +++- .../SdkContentTests.cs | 5 +- .../SkippableFactAttribute.cs | 53 -------- .../SkippableTheoryAttribute.cs | 19 --- .../SourceBuiltArtifactsTests.cs | 4 +- .../Utilities.cs | 8 -- .../Utilities.cs | 82 ------------- 26 files changed, 230 insertions(+), 368 deletions(-) create mode 100644 src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalFactAttribute.cs create mode 100644 src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalTheoryAttribute.cs delete mode 100644 src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Directory.Build.props delete mode 100644 src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.sln delete mode 100644 src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableFactAttribute.cs delete mode 100644 src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableTheoryAttribute.cs diff --git a/src/SourceBuild/content/.gitignore b/src/SourceBuild/content/.gitignore index e88b2a3e8ead..1f77e38b1215 100644 --- a/src/SourceBuild/content/.gitignore +++ b/src/SourceBuild/content/.gitignore @@ -3,7 +3,4 @@ /artifacts /prereqs/packages /src/nuget-client/NuGet.config -/test/Microsoft.DotNet.SourceBuild.SmokeTests/bin -/test/Microsoft.DotNet.SourceBuild.SmokeTests/obj -/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults *.binlog diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index b3265733ede7..e63f45bd6e43 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -213,6 +213,7 @@ .prebuilt.xml $(PackageReportDir)poison-catalog.xml $(PackageReportDir)poisoned.txt + $(PackageReportDir)poison-usage.xml diff --git a/src/SourceBuild/content/Directory.Packages.props b/src/SourceBuild/content/Directory.Packages.props index 398641fa5b5c..a0d4c95cc1f4 100644 --- a/src/SourceBuild/content/Directory.Packages.props +++ b/src/SourceBuild/content/Directory.Packages.props @@ -16,6 +16,8 @@ + + diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index c6d8d325ba8f..3c245ae6ce47 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -1,15 +1,14 @@ - - + - - - netstandard2.0 - true - + + + + + + - - - + source-build non-source-build @@ -17,12 +16,10 @@ + - + + @@ -48,8 +45,8 @@ - - + + diff --git a/src/SourceBuild/content/build.sh b/src/SourceBuild/content/build.sh index a56de7c8fc7f..c7c1f63c4cd4 100755 --- a/src/SourceBuild/content/build.sh +++ b/src/SourceBuild/content/build.sh @@ -113,7 +113,7 @@ while [[ $# > 0 ]]; do ;; -test|-t) export NUGET_PACKAGES=$NUGET_PACKAGES/smoke-tests - properties="$properties /t:RunSmokeTest" + properties="$properties /t:Test" test=true ;; diff --git a/src/SourceBuild/content/eng/Versions.props b/src/SourceBuild/content/eng/Versions.props index 6bb590a6bda5..57d207aeaaf4 100644 --- a/src/SourceBuild/content/eng/Versions.props +++ b/src/SourceBuild/content/eng/Versions.props @@ -25,13 +25,15 @@ --> 9.0.100-preview.4.24179.1 9.0.100-preview.4.24179.1 - + + 2.0.0-beta4.24126.1 + 17.8.3 - + 9.0.0-preview.2.24128.5 9.0.0-preview.2.24128.5 9.0.0-preview.2.24128.5 - - 2.0.0-beta4.24126.1 + + 13.0.3 diff --git a/src/SourceBuild/content/eng/build.sourcebuild.targets b/src/SourceBuild/content/eng/build.sourcebuild.targets index 557e4c78d497..349076b6756a 100644 --- a/src/SourceBuild/content/eng/build.sourcebuild.targets +++ b/src/SourceBuild/content/eng/build.sourcebuild.targets @@ -1,13 +1,5 @@ - - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'test', 'Microsoft.DotNet.SourceBuild.SmokeTests')) - - - - $(PackageReportDir)poison-usage.xml - - - - - - - - - - SMOKE_TESTS_POISON_REPORT_PATH=$(PoisonUsageReportFile); - %(SdkTarballItem.Identity) - %(SourceBuiltArtifactsItem.Identity) - normal - - - - - - - - - - - - $(SmokeTestsDir)bin/$(Configuration)/$(NetCurrent)/ - $(SmokeTestsArtifactsDir)packages/ - - - - - - - - - - - - $(ArtifactsAssetsDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) - $(SmokeTestsArtifactsDir)prereq-packages/ - - - - - - - - - - - - - diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs index e8cafd64183c..7ac8005c67a0 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs @@ -42,13 +42,9 @@ public ArtifactsSizeTest(ITestOutputHelper outputHelper) : base(outputHelper) } } - [SkippableFact(Config.IncludeArtifactsSizeEnv, skipOnFalseEnv: true)] + [ConditionalFact(typeof(Config), nameOf(IncludeArtifactsSize))] public void CompareArtifactsToBaseline() { - Utilities.ValidateNotNullOrWhiteSpace(Config.SourceBuiltArtifactsPath, Config.SourceBuiltArtifactsPathEnv); - Utilities.ValidateNotNullOrWhiteSpace(Config.SdkTarballPath, Config.SdkTarballPathEnv); - Utilities.ValidateNotNullOrWhiteSpace(Config.TargetRid, Config.TargetRidEnv); - var tarEntries = ProcessSdkAndArtifactsTarballs(); ScanForDifferences(tarEntries); UpdateBaselineFile(); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalFactAttribute.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalFactAttribute.cs new file mode 100644 index 000000000000..3df84b245a56 --- /dev/null +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalFactAttribute.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Reflection; +using Xunit; + +namespace Microsoft.DotNet.SourceBuild.SmokeTests; + +/// +/// A Fact that conditionally runs based on a type's boolean property member +/// +internal sealed class ConditionalFactAttribute : FactAttribute +{ + public ConditionalFactAttribute(Type calleeType, string memberName, string? reason = null) + { + EvaluateSkip(calleeType, memberName, reason, (skip) => Skip = skip); + } + + internal static void EvaluateSkip(Type calleeType, string memberName, string? reason = null, Action setSkip) + { + TypeInfo typeInfo = calleeType.GetTypeInfo(); + bool shouldRun = (bool?)typeInfo.GetProperty(memberName)?.GetValue(null) ?? false; + if (!shouldRun) + { + setSkip(reason ?? "Skipped"); + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalTheoryAttribute.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalTheoryAttribute.cs new file mode 100644 index 000000000000..1c6a276feb4f --- /dev/null +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalTheoryAttribute.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Xunit; + +namespace Microsoft.DotNet.SourceBuild.SmokeTests; + +/// +/// A Theory that conditionally runs based on a type's boolean property member. +/// +internal sealed class ConditionalTheoryAttribute : TheoryAttribute +{ + public ConditionalTheoryAttribute(Type calleeType, string memberName, string? reason = null) + { + ConditionalFactAttribute.EvaluateSkip(calleeType, memberName, reason, (skip) => Skip = skip); + } +} diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs index 6bc2393e89fa..5b232b5141df 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs @@ -9,43 +9,28 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests; internal static class Config { - public const string DotNetDirectoryEnv = "SMOKE_TESTS_DOTNET_DIR"; - public const string ExcludeOmniSharpEnv = "SMOKE_TESTS_EXCLUDE_OMNISHARP"; - public const string IncludeArtifactsSizeEnv = "SMOKE_TESTS_INCLUDE_ARTIFACTSSIZE"; - public const string MsftSdkTarballPathEnv = "SMOKE_TESTS_MSFT_SDK_TARBALL_PATH"; - public const string PoisonReportPathEnv = "SMOKE_TESTS_POISON_REPORT_PATH"; - public const string PortableRidEnv = "SMOKE_TESTS_PORTABLE_RID"; - public const string PrereqsPathEnv = "SMOKE_TESTS_PREREQS_PATH"; - public const string CustomPackagesPathEnv = "SMOKE_TESTS_CUSTOM_PACKAGES_PATH"; - public const string SdkTarballPathEnv = "SMOKE_TESTS_SDK_TARBALL_PATH"; - public const string SourceBuiltArtifactsPathEnv = "SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH"; - public const string TargetRidEnv = "SMOKE_TESTS_TARGET_RID"; - public const string WarnSdkContentDiffsEnv = "SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS"; - public const string WarnLicenseScanDiffsEnv = "SMOKE_TESTS_WARN_LICENSE_SCAN_DIFFS"; - public const string RunningInCIEnv = "SMOKE_TESTS_RUNNING_IN_CI"; - public const string LicenseScanPathEnv = "SMOKE_TESTS_LICENSE_SCAN_PATH"; + const string ConfigSwitchPrefix = "Microsoft.DotNet.SourceBuild.SmokeTests."; - public static string DotNetDirectory { get; } = - Environment.GetEnvironmentVariable(DotNetDirectoryEnv) ?? Path.Combine(Directory.GetCurrentDirectory(), ".dotnet"); - public static string? MsftSdkTarballPath { get; } = Environment.GetEnvironmentVariable(MsftSdkTarballPathEnv); - public static string? PoisonReportPath { get; } = Environment.GetEnvironmentVariable(PoisonReportPathEnv); - public static string PortableRid { get; } = Environment.GetEnvironmentVariable(PortableRidEnv) ?? - throw new InvalidOperationException($"'{Config.PortableRidEnv}' must be specified"); - public static string? PrereqsPath { get; } = Environment.GetEnvironmentVariable(PrereqsPathEnv); - public static string? CustomPackagesPath { get; } = Environment.GetEnvironmentVariable(CustomPackagesPathEnv); - public static string? SdkTarballPath { get; } = Environment.GetEnvironmentVariable(SdkTarballPathEnv); - public static string? SourceBuiltArtifactsPath { get; } = Environment.GetEnvironmentVariable(SourceBuiltArtifactsPathEnv); - public static string TargetRid { get; } = Environment.GetEnvironmentVariable(TargetRidEnv) ?? - throw new InvalidOperationException($"'{Config.TargetRidEnv}' must be specified"); - public static string TargetArchitecture { get; } = TargetRid.Split('-')[1]; - public static bool WarnOnSdkContentDiffs { get; } = - bool.TryParse(Environment.GetEnvironmentVariable(WarnSdkContentDiffsEnv), out bool warnOnSdkContentDiffs) && warnOnSdkContentDiffs; - public static bool WarnOnLicenseScanDiffs { get; } = - bool.TryParse(Environment.GetEnvironmentVariable(WarnLicenseScanDiffsEnv), out bool warnOnLicenseScanDiffs) && warnOnLicenseScanDiffs; + public static string DotNetDirectory => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(DotNetDirectory)) ?? throw new InvalidOperationException("DotNetDirectory must be specified"); + public static string PortableRid => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PortableRid)) ?? throw new InvalidOperationException("Portable RID must be specified"); + public static string TargetRid => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(TargetRid)) ?? throw new InvalidOperationException("Target RID must be specified"); + + public static string? CustomPackagesPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(CustomPackagesPath)); + public static bool ExcludeOmniSharpTests => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(ExcludeOmniSharpTests)), out bool excludeOmniSharpTests) && excludeOmniSharpTests; + public static bool IncludeArtifactsSize => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(IncludeArtifactsSize)), out bool includeArtifactsSize) && includeArtifactsSize; + public static string? LicenseScanPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(LicenseScanPath)); + public static string? MsftSdkTarballPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(MsftSdkTarballPath)); + public static string? PoisonReportPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PoisonReportPath)); + public static string? PrereqsPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PrereqsPath)); + public static string? SdkTarballPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(SdkTarballPath)); + public static string? SourceBuiltArtifactsPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(SourceBuiltArtifactsPath)); + public static bool WarnOnLicenseScanDiffs => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(WarnOnLicenseScanDiffs)), out bool warnOnLicenseScanDiffs) && warnOnLicenseScanDiffs; + public static bool WarnOnSdkContentDiffs => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(WarnOnSdkContentDiffs)), out bool warnOnSdkContentDiffs) && warnOnSdkContentDiffs; // Indicates whether the tests are being run in the context of a CI pipeline - public static bool RunningInCI { get; } = - bool.TryParse(Environment.GetEnvironmentVariable(RunningInCIEnv), out bool runningInCI) && runningInCI; - - public static string? LicenseScanPath { get; } = Environment.GetEnvironmentVariable(LicenseScanPathEnv); + public static bool RunningInCI => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOTNET_CI")) || + !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT")) || + !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")); + + public static string TargetArchitecture => TargetRid.Split('-')[1]; } diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Directory.Build.props b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Directory.Build.props deleted file mode 100644 index ae5a2637d3e5..000000000000 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Directory.Build.props +++ /dev/null @@ -1,10 +0,0 @@ - - - - - false - false - - - diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs index 291d492ed1a8..817a41a32074 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs @@ -72,7 +72,7 @@ private static void InitNugetConfig() if (!Directory.Exists(Config.PrereqsPath)) { throw new InvalidOperationException( - $"Prereqs path '{Config.PrereqsPath}' specified in {Config.PrereqsPathEnv} does not exist."); + $"Prereqs path '{Config.PrereqsPath}' specified via /p:SmokeTestsPrereqsPath='...' does not exist."); } string nugetConfig = File.ReadAllText(nugetConfigPath); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs index 4cab87bc84d7..a80af9648da3 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs @@ -112,13 +112,15 @@ public class LicenseScanTests : TestBase private readonly string _targetRepo; + public bool ShouldScanForLicenses => !string.IsNullOrWhiteSpace(Config.LicenseScanPath); + public LicenseScanTests(ITestOutputHelper outputHelper) : base(outputHelper) { Assert.NotNull(Config.LicenseScanPath); _targetRepo = new DirectoryInfo(Config.LicenseScanPath).Name; } - [SkippableFact(Config.LicenseScanPathEnv, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(typeof(LicenseScanTests), nameof(ShouldScanForLicenses))] public void ScanForLicenses() { Assert.NotNull(Config.LicenseScanPath); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj index 80f1717e7575..139f965a0b77 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj @@ -1,20 +1,19 @@ - net9.0 + $(NetCurrent) enable - false + + true $(DefaultExcludesInProjectFolder);assets/**/* + + false - - - - - - - + + @@ -22,10 +21,99 @@ CopyToOutputDirectory="Always" /> - - - + + + + + + + %(SourceBuiltArtifactsItem.Identity) + + + + + + $(TargetDir)extracted-sdk + + + $(TargetRid) + + + $(PortableRid) + + + + + $(SmokeTestsCustomSourceBuiltPackagesPath) + + + $(SmokeTestsExcludeOmniSharpTests) + + + $(SmokeTestsIncludeArtifactsSize) + + + $(MsftSdkTarballPath) + + + $(PoisonUsageReportFile) + + + $(SmokeTestsPrereqsPath) + + + $(SdkTarballPath) + + + $(SourceBuiltArtifactsPath) + + + $(SmokeTestsWarnOnLicenseScanDiffs) + + + $(SmokeTestsWarnOnSdkContentDiffs) + + + + + + + + $(TargetDir)packages/ + $(ArtifactsAssetsDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) + + + + + + + + + + + + + $(IntermediateOutputPath)prereq-packages/ + + + + + + + + + diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.sln b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.sln deleted file mode 100644 index 714ed63b6e67..000000000000 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.sln +++ /dev/null @@ -1,25 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.32112.339 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.SourceBuild.SmokeTests", "Microsoft.DotNet.SourceBuild.SmokeTests.csproj", "{FEB5A0B5-460B-432A-BED8-243557188AFF}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {FEB5A0B5-460B-432A-BED8-243557188AFF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FEB5A0B5-460B-432A-BED8-243557188AFF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FEB5A0B5-460B-432A-BED8-243557188AFF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FEB5A0B5-460B-432A-BED8-243557188AFF}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {75929D76-EEB5-4793-8335-DF623CC72B56} - EndGlobalSection -EndGlobal diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs index e140df73ba62..48198cbd9458 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs @@ -23,9 +23,11 @@ public class OmniSharpTests : SdkTests private string OmniSharpDirectory { get; } = Path.Combine(Directory.GetCurrentDirectory(), nameof(OmniSharpTests)); + public bool ShouldVerifyScenario => !Config.ExcludeOmniSharp && Config.TargetArchitecture != "ppc64le" && Config.TargetArchitecture != "s390x"; + public OmniSharpTests(ITestOutputHelper outputHelper) : base(outputHelper) { } - [SkippableTheory(Config.ExcludeOmniSharpEnv, skipOnTrueEnv: true, skipArchitectures: new[] { "ppc64le", "s390x" })] + [ConditionalTheoryAttribute(typeof(OmniSharpTests), nameof(ShouldVerifyScenario))] [InlineData(DotNetTemplate.BlazorWasm)] [InlineData(DotNetTemplate.ClassLib)] [InlineData(DotNetTemplate.Console)] diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/PoisonTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/PoisonTests.cs index 10a794fc412e..eccf5de267be 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/PoisonTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/PoisonTests.cs @@ -11,9 +11,11 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests { public class PoisonTests : SdkTests { + public bool ShouldRunPoisonTests => !string.IsNullOrWhiteSpace(Config.PoisonReportPath); + public PoisonTests(ITestOutputHelper outputHelper) : base(outputHelper) { } - [SkippableFact(Config.PoisonReportPathEnv, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(PoisonTests), nameof(ShouldRunPoisonTests))] public void VerifyUsage() { if (!File.Exists(Config.PoisonReportPath)) diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md index 9e9586075efc..7301a29e04c6 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md @@ -1,7 +1,22 @@ # Source Build Smoke Tests -* Run these tests via `build.sh --run-smoke-test` -* Various configuration settings are stored in `Config.cs` +* Run these tests via `build.sh --projects test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj --test` + +The following properties are automatically available during test execution but can be overwritten: +- MsftSdkTarballPath +- PoisonUsageReportFile +- SdkTarballPath +- SourceBuiltArtifactsPath + +Optional msbuild properties: +- SmokeTestsCustomSourceBuiltPackagesPath +- SmokeTestsExcludeOmniSharpTests +- SmokeTestsIncludeArtifactsSize +- SmokeTestsPrereqsPath +- SmokeTestsWarnOnLicenseScanDiffs +- SmokeTestsWarnOnSdkContentDiffs + +Make sure to rebuild the test project when changing one of those values. ## Dependencies @@ -15,5 +30,5 @@ The following programs are used by some tests: ## Prereq Packages Some prerelease scenarios, usually security updates, require non-source-built packages which are not publicly available. -Specify the directory where these packages can be found via the `SMOKE_TESTS_PREREQS_PATH` environment variable when running tests via `build.sh --run-smoke-test` e.g. -`SMOKE_TESTS_PREREQS_PATH=prereqs/packages/smoke-test-prereqs`. +Specify the directory where these packages can be found via the `SmokeTestsPrereqsPath` msbuild property when running tests via `build.sh --projects test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj --test` e.g. +`/p:SmokeTestsPrereqsPath=prereqs/packages/smoke-test-prereqs`. \ No newline at end of file diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs index 29bb7dbc55ca..47bbf542bb48 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs @@ -21,6 +21,7 @@ public class SdkContentTests : SdkTests { private const string MsftSdkType = "msft"; private const string SourceBuildSdkType = "sb"; + public bool ShouldCompareSdkContent => !string.IsNullOrWhiteSpace(Config.MsftSdkTarballPath) && !string.IsNullOrWhiteSpace(Config.SdkTarballPathEnv); public SdkContentTests(ITestOutputHelper outputHelper) : base(outputHelper) { } @@ -31,7 +32,7 @@ public SdkContentTests(ITestOutputHelper outputHelper) : base(outputHelper) { } /// This makes the baseline durable between releases. This does mean however, entries /// in the baseline may appear identical if the diff is version specific. /// - [SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.SdkTarballPathEnv }, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(SdkContentTests), nameof(ShouldCompareSdkContent))] public void CompareMsftToSbFileList() { const string msftFileListingFileName = "msftSdkFiles.txt"; @@ -44,7 +45,7 @@ public void CompareMsftToSbFileList() BaselineHelper.CompareBaselineContents("MsftToSbSdkFiles.diff", diff, OutputHelper, Config.WarnOnSdkContentDiffs); } - [SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.SdkTarballPathEnv }, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(SdkContentTests), nameof(ShouldCompareSdkContent))] public void CompareMsftToSbAssemblyVersions() { Assert.NotNull(Config.MsftSdkTarballPath); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableFactAttribute.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableFactAttribute.cs deleted file mode 100644 index c1314e19e8b5..000000000000 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableFactAttribute.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Linq; -using Xunit; - -namespace Microsoft.DotNet.SourceBuild.SmokeTests; - -/// -/// A Fact that will be skipped based on the specified environment variable's value. -/// -internal class SkippableFactAttribute : FactAttribute -{ - public SkippableFactAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) => - EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envName); - - public SkippableFactAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) => - EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envNames); - - public static void EvaluateSkips(bool skipOnNullOrWhiteSpaceEnv, bool skipOnTrueEnv, bool skipOnFalseEnv, string[] skipArchitectures, Action setSkip, params string[] envNames) - { - foreach (string envName in envNames) - { - string? envValue = Environment.GetEnvironmentVariable(envName); - - if (skipOnNullOrWhiteSpaceEnv && string.IsNullOrWhiteSpace(envValue)) - { - setSkip($"Skipping because `{envName}` is null or whitespace"); - break; - } - else if (skipOnTrueEnv && bool.TryParse(envValue, out bool boolValue) && boolValue) - { - setSkip($"Skipping because `{envName}` is set to True"); - break; - } - else if (skipOnFalseEnv && (!bool.TryParse(envValue, out boolValue) || !boolValue)) - { - setSkip($"Skipping because `{envName}` is set to False or an invalid value"); - break; - } - } - - if (skipArchitectures != null) { - string? arch = Config.TargetArchitecture; - if (skipArchitectures.Contains(arch)) - { - setSkip($"Skipping because arch is `{arch}`"); - } - } - } -} diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableTheoryAttribute.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableTheoryAttribute.cs deleted file mode 100644 index a43228c0d2a6..000000000000 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableTheoryAttribute.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Xunit; - -namespace Microsoft.DotNet.SourceBuild.SmokeTests; - -/// -/// A Theory that will be skipped based on the specified environment variable's value. -/// -internal class SkippableTheoryAttribute : TheoryAttribute -{ - public SkippableTheoryAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) => - SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envName); - - public SkippableTheoryAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) => - SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envNames); -} diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs index 7d6278b80b53..460004b8222d 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs @@ -15,9 +15,11 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests; public class SourceBuiltArtifactsTests : SdkTests { + public bool ShouldVerifyVersionFile => !string.IsNullOrWhiteSpace(Config.SourceBuiltArtifactsPath); + public SourceBuiltArtifactsTests(ITestOutputHelper outputHelper) : base(outputHelper) { } - [SkippableFact(Config.SourceBuiltArtifactsPathEnv, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(SourceBuiltArtifactsTests), nameof(ShouldVerifyVersionFile))] public void VerifyVersionFile() { Assert.NotNull(Config.SourceBuiltArtifactsPath); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs index 228a65a61e94..e069e36206af 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs @@ -160,14 +160,6 @@ public static void LogWarningMessage(this ITestOutputHelper outputHelper, string outputHelper.WriteLine("##vso[task.complete result=SucceededWithIssues;]"); } - public static void ValidateNotNullOrWhiteSpace(string? variable, string variableName) - { - if (string.IsNullOrWhiteSpace(variable)) - { - throw new ArgumentException($"{variableName} is null, empty, or whitespace."); - } - } - public static string GetFile(string path, string pattern) { string[] files = Directory.GetFiles(path, pattern, SearchOption.AllDirectories); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Utilities.cs b/src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Utilities.cs index a56b4f0259ed..16b0da345804 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Utilities.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Utilities.cs @@ -141,86 +141,4 @@ public static IEnumerable GetTarballContentNames(string tarballPath) throw new InvalidOperationException($"Unsupported archive format: {tarballPath}"); } } - - public static void ExtractNupkg(string package, string outputDir) - { - Directory.CreateDirectory(outputDir); - - using ZipArchive zip = ZipFile.OpenRead(package); - foreach (ZipArchiveEntry entry in zip.Entries) - { - string outputPath = Path.Combine(outputDir, entry.FullName); - Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); - entry.ExtractToFile(outputPath); - } - } - - public static async Task RetryAsync(Func executor, ITestOutputHelper outputHelper) - { - await Utilities.RetryAsync( - async () => - { - try - { - await executor(); - return null; - } - catch (Exception e) - { - return e; - } - }, - outputHelper); - } - - private static async Task RetryAsync(Func> executor, ITestOutputHelper outputHelper) - { - const int maxRetries = 5; - const int waitFactor = 5; - - int retryCount = 0; - - Exception? exception = await executor(); - while (exception != null) - { - retryCount++; - if (retryCount >= maxRetries) - { - throw new InvalidOperationException($"Failed after {retryCount} retries.", exception); - } - - int waitTime = Convert.ToInt32(Math.Pow(waitFactor, retryCount - 1)); - if (outputHelper != null) - { - outputHelper.WriteLine($"Retry {retryCount}/{maxRetries}, retrying in {waitTime} seconds..."); - } - - Thread.Sleep(TimeSpan.FromSeconds(waitTime)); - exception = await executor(); - } - } - - public static void LogWarningMessage(this ITestOutputHelper outputHelper, string message) - { - string prefix = "##vso[task.logissue type=warning;]"; - - outputHelper.WriteLine($"{Environment.NewLine}{prefix}{message}.{Environment.NewLine}"); - outputHelper.WriteLine("##vso[task.complete result=SucceededWithIssues;]"); - } - - public static void ValidateNotNullOrWhiteSpace(string? variable, string variableName) - { - if (string.IsNullOrWhiteSpace(variable)) - { - throw new ArgumentException($"{variableName} is null, empty, or whitespace."); - } - } - - public static string GetFile(string path, string pattern) - { - string[] files = Directory.GetFiles(path, pattern, SearchOption.AllDirectories); - Assert.False(files.Length > 1, $"Found multiple files matching the pattern {pattern}: {Environment.NewLine}{string.Join(Environment.NewLine, files)}"); - Assert.False(files.Length == 0, $"Did not find any files matching the pattern {pattern}"); - return files[0]; - } }