diff --git a/src/SourceBuild/content/.gitignore b/src/SourceBuild/content/.gitignore index e88b2a3e8ead..1f77e38b1215 100644 --- a/src/SourceBuild/content/.gitignore +++ b/src/SourceBuild/content/.gitignore @@ -3,7 +3,4 @@ /artifacts /prereqs/packages /src/nuget-client/NuGet.config -/test/Microsoft.DotNet.SourceBuild.SmokeTests/bin -/test/Microsoft.DotNet.SourceBuild.SmokeTests/obj -/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults *.binlog diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index b3265733ede7..e63f45bd6e43 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -213,6 +213,7 @@ .prebuilt.xml $(PackageReportDir)poison-catalog.xml $(PackageReportDir)poisoned.txt + $(PackageReportDir)poison-usage.xml diff --git a/src/SourceBuild/content/Directory.Packages.props b/src/SourceBuild/content/Directory.Packages.props index 398641fa5b5c..a0d4c95cc1f4 100644 --- a/src/SourceBuild/content/Directory.Packages.props +++ b/src/SourceBuild/content/Directory.Packages.props @@ -16,6 +16,8 @@ + + diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index c6d8d325ba8f..3c245ae6ce47 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -1,15 +1,14 @@ - - + - - - netstandard2.0 - true - + + + + + + - - - + source-build non-source-build @@ -17,12 +16,10 @@ + - + + @@ -48,8 +45,8 @@ - - + + diff --git a/src/SourceBuild/content/build.sh b/src/SourceBuild/content/build.sh index a56de7c8fc7f..c7c1f63c4cd4 100755 --- a/src/SourceBuild/content/build.sh +++ b/src/SourceBuild/content/build.sh @@ -113,7 +113,7 @@ while [[ $# > 0 ]]; do ;; -test|-t) export NUGET_PACKAGES=$NUGET_PACKAGES/smoke-tests - properties="$properties /t:RunSmokeTest" + properties="$properties /t:Test" test=true ;; diff --git a/src/SourceBuild/content/eng/Versions.props b/src/SourceBuild/content/eng/Versions.props index 6bb590a6bda5..57d207aeaaf4 100644 --- a/src/SourceBuild/content/eng/Versions.props +++ b/src/SourceBuild/content/eng/Versions.props @@ -25,13 +25,15 @@ --> 9.0.100-preview.4.24179.1 9.0.100-preview.4.24179.1 - + + 2.0.0-beta4.24126.1 + 17.8.3 - + 9.0.0-preview.2.24128.5 9.0.0-preview.2.24128.5 9.0.0-preview.2.24128.5 - - 2.0.0-beta4.24126.1 + + 13.0.3 diff --git a/src/SourceBuild/content/eng/build.sourcebuild.targets b/src/SourceBuild/content/eng/build.sourcebuild.targets index 557e4c78d497..349076b6756a 100644 --- a/src/SourceBuild/content/eng/build.sourcebuild.targets +++ b/src/SourceBuild/content/eng/build.sourcebuild.targets @@ -1,13 +1,5 @@ - - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'test', 'Microsoft.DotNet.SourceBuild.SmokeTests')) - - - - $(PackageReportDir)poison-usage.xml - - - - - - - - - - SMOKE_TESTS_POISON_REPORT_PATH=$(PoisonUsageReportFile); - %(SdkTarballItem.Identity) - %(SourceBuiltArtifactsItem.Identity) - normal - - - - - - - - - - - - $(SmokeTestsDir)bin/$(Configuration)/$(NetCurrent)/ - $(SmokeTestsArtifactsDir)packages/ - - - - - - - - - - - - $(ArtifactsAssetsDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) - $(SmokeTestsArtifactsDir)prereq-packages/ - - - - - - - - - - - - - diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs index e8cafd64183c..7ac8005c67a0 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs @@ -42,13 +42,9 @@ public ArtifactsSizeTest(ITestOutputHelper outputHelper) : base(outputHelper) } } - [SkippableFact(Config.IncludeArtifactsSizeEnv, skipOnFalseEnv: true)] + [ConditionalFact(typeof(Config), nameOf(IncludeArtifactsSize))] public void CompareArtifactsToBaseline() { - Utilities.ValidateNotNullOrWhiteSpace(Config.SourceBuiltArtifactsPath, Config.SourceBuiltArtifactsPathEnv); - Utilities.ValidateNotNullOrWhiteSpace(Config.SdkTarballPath, Config.SdkTarballPathEnv); - Utilities.ValidateNotNullOrWhiteSpace(Config.TargetRid, Config.TargetRidEnv); - var tarEntries = ProcessSdkAndArtifactsTarballs(); ScanForDifferences(tarEntries); UpdateBaselineFile(); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalFactAttribute.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalFactAttribute.cs new file mode 100644 index 000000000000..3df84b245a56 --- /dev/null +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalFactAttribute.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Reflection; +using Xunit; + +namespace Microsoft.DotNet.SourceBuild.SmokeTests; + +/// +/// A Fact that conditionally runs based on a type's boolean property member +/// +internal sealed class ConditionalFactAttribute : FactAttribute +{ + public ConditionalFactAttribute(Type calleeType, string memberName, string? reason = null) + { + EvaluateSkip(calleeType, memberName, reason, (skip) => Skip = skip); + } + + internal static void EvaluateSkip(Type calleeType, string memberName, string? reason = null, Action setSkip) + { + TypeInfo typeInfo = calleeType.GetTypeInfo(); + bool shouldRun = (bool?)typeInfo.GetProperty(memberName)?.GetValue(null) ?? false; + if (!shouldRun) + { + setSkip(reason ?? "Skipped"); + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalTheoryAttribute.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalTheoryAttribute.cs new file mode 100644 index 000000000000..1c6a276feb4f --- /dev/null +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalTheoryAttribute.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Xunit; + +namespace Microsoft.DotNet.SourceBuild.SmokeTests; + +/// +/// A Theory that conditionally runs based on a type's boolean property member. +/// +internal sealed class ConditionalTheoryAttribute : TheoryAttribute +{ + public ConditionalTheoryAttribute(Type calleeType, string memberName, string? reason = null) + { + ConditionalFactAttribute.EvaluateSkip(calleeType, memberName, reason, (skip) => Skip = skip); + } +} diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs index 6bc2393e89fa..5b232b5141df 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs @@ -9,43 +9,28 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests; internal static class Config { - public const string DotNetDirectoryEnv = "SMOKE_TESTS_DOTNET_DIR"; - public const string ExcludeOmniSharpEnv = "SMOKE_TESTS_EXCLUDE_OMNISHARP"; - public const string IncludeArtifactsSizeEnv = "SMOKE_TESTS_INCLUDE_ARTIFACTSSIZE"; - public const string MsftSdkTarballPathEnv = "SMOKE_TESTS_MSFT_SDK_TARBALL_PATH"; - public const string PoisonReportPathEnv = "SMOKE_TESTS_POISON_REPORT_PATH"; - public const string PortableRidEnv = "SMOKE_TESTS_PORTABLE_RID"; - public const string PrereqsPathEnv = "SMOKE_TESTS_PREREQS_PATH"; - public const string CustomPackagesPathEnv = "SMOKE_TESTS_CUSTOM_PACKAGES_PATH"; - public const string SdkTarballPathEnv = "SMOKE_TESTS_SDK_TARBALL_PATH"; - public const string SourceBuiltArtifactsPathEnv = "SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH"; - public const string TargetRidEnv = "SMOKE_TESTS_TARGET_RID"; - public const string WarnSdkContentDiffsEnv = "SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS"; - public const string WarnLicenseScanDiffsEnv = "SMOKE_TESTS_WARN_LICENSE_SCAN_DIFFS"; - public const string RunningInCIEnv = "SMOKE_TESTS_RUNNING_IN_CI"; - public const string LicenseScanPathEnv = "SMOKE_TESTS_LICENSE_SCAN_PATH"; + const string ConfigSwitchPrefix = "Microsoft.DotNet.SourceBuild.SmokeTests."; - public static string DotNetDirectory { get; } = - Environment.GetEnvironmentVariable(DotNetDirectoryEnv) ?? Path.Combine(Directory.GetCurrentDirectory(), ".dotnet"); - public static string? MsftSdkTarballPath { get; } = Environment.GetEnvironmentVariable(MsftSdkTarballPathEnv); - public static string? PoisonReportPath { get; } = Environment.GetEnvironmentVariable(PoisonReportPathEnv); - public static string PortableRid { get; } = Environment.GetEnvironmentVariable(PortableRidEnv) ?? - throw new InvalidOperationException($"'{Config.PortableRidEnv}' must be specified"); - public static string? PrereqsPath { get; } = Environment.GetEnvironmentVariable(PrereqsPathEnv); - public static string? CustomPackagesPath { get; } = Environment.GetEnvironmentVariable(CustomPackagesPathEnv); - public static string? SdkTarballPath { get; } = Environment.GetEnvironmentVariable(SdkTarballPathEnv); - public static string? SourceBuiltArtifactsPath { get; } = Environment.GetEnvironmentVariable(SourceBuiltArtifactsPathEnv); - public static string TargetRid { get; } = Environment.GetEnvironmentVariable(TargetRidEnv) ?? - throw new InvalidOperationException($"'{Config.TargetRidEnv}' must be specified"); - public static string TargetArchitecture { get; } = TargetRid.Split('-')[1]; - public static bool WarnOnSdkContentDiffs { get; } = - bool.TryParse(Environment.GetEnvironmentVariable(WarnSdkContentDiffsEnv), out bool warnOnSdkContentDiffs) && warnOnSdkContentDiffs; - public static bool WarnOnLicenseScanDiffs { get; } = - bool.TryParse(Environment.GetEnvironmentVariable(WarnLicenseScanDiffsEnv), out bool warnOnLicenseScanDiffs) && warnOnLicenseScanDiffs; + public static string DotNetDirectory => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(DotNetDirectory)) ?? throw new InvalidOperationException("DotNetDirectory must be specified"); + public static string PortableRid => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PortableRid)) ?? throw new InvalidOperationException("Portable RID must be specified"); + public static string TargetRid => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(TargetRid)) ?? throw new InvalidOperationException("Target RID must be specified"); + + public static string? CustomPackagesPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(CustomPackagesPath)); + public static bool ExcludeOmniSharpTests => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(ExcludeOmniSharpTests)), out bool excludeOmniSharpTests) && excludeOmniSharpTests; + public static bool IncludeArtifactsSize => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(IncludeArtifactsSize)), out bool includeArtifactsSize) && includeArtifactsSize; + public static string? LicenseScanPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(LicenseScanPath)); + public static string? MsftSdkTarballPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(MsftSdkTarballPath)); + public static string? PoisonReportPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PoisonReportPath)); + public static string? PrereqsPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PrereqsPath)); + public static string? SdkTarballPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(SdkTarballPath)); + public static string? SourceBuiltArtifactsPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(SourceBuiltArtifactsPath)); + public static bool WarnOnLicenseScanDiffs => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(WarnOnLicenseScanDiffs)), out bool warnOnLicenseScanDiffs) && warnOnLicenseScanDiffs; + public static bool WarnOnSdkContentDiffs => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(WarnOnSdkContentDiffs)), out bool warnOnSdkContentDiffs) && warnOnSdkContentDiffs; // Indicates whether the tests are being run in the context of a CI pipeline - public static bool RunningInCI { get; } = - bool.TryParse(Environment.GetEnvironmentVariable(RunningInCIEnv), out bool runningInCI) && runningInCI; - - public static string? LicenseScanPath { get; } = Environment.GetEnvironmentVariable(LicenseScanPathEnv); + public static bool RunningInCI => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOTNET_CI")) || + !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT")) || + !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")); + + public static string TargetArchitecture => TargetRid.Split('-')[1]; } diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Directory.Build.props b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Directory.Build.props deleted file mode 100644 index ae5a2637d3e5..000000000000 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Directory.Build.props +++ /dev/null @@ -1,10 +0,0 @@ - - - - - false - false - - - diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs index 291d492ed1a8..817a41a32074 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs @@ -72,7 +72,7 @@ private static void InitNugetConfig() if (!Directory.Exists(Config.PrereqsPath)) { throw new InvalidOperationException( - $"Prereqs path '{Config.PrereqsPath}' specified in {Config.PrereqsPathEnv} does not exist."); + $"Prereqs path '{Config.PrereqsPath}' specified via /p:SmokeTestsPrereqsPath='...' does not exist."); } string nugetConfig = File.ReadAllText(nugetConfigPath); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs index 4cab87bc84d7..a80af9648da3 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs @@ -112,13 +112,15 @@ public class LicenseScanTests : TestBase private readonly string _targetRepo; + public bool ShouldScanForLicenses => !string.IsNullOrWhiteSpace(Config.LicenseScanPath); + public LicenseScanTests(ITestOutputHelper outputHelper) : base(outputHelper) { Assert.NotNull(Config.LicenseScanPath); _targetRepo = new DirectoryInfo(Config.LicenseScanPath).Name; } - [SkippableFact(Config.LicenseScanPathEnv, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(typeof(LicenseScanTests), nameof(ShouldScanForLicenses))] public void ScanForLicenses() { Assert.NotNull(Config.LicenseScanPath); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj index 80f1717e7575..139f965a0b77 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj @@ -1,20 +1,19 @@ - net9.0 + $(NetCurrent) enable - false + + true $(DefaultExcludesInProjectFolder);assets/**/* + + false - - - - - - - + + @@ -22,10 +21,99 @@ CopyToOutputDirectory="Always" /> - - - + + + + + + + %(SourceBuiltArtifactsItem.Identity) + + + + + + $(TargetDir)extracted-sdk + + + $(TargetRid) + + + $(PortableRid) + + + + + $(SmokeTestsCustomSourceBuiltPackagesPath) + + + $(SmokeTestsExcludeOmniSharpTests) + + + $(SmokeTestsIncludeArtifactsSize) + + + $(MsftSdkTarballPath) + + + $(PoisonUsageReportFile) + + + $(SmokeTestsPrereqsPath) + + + $(SdkTarballPath) + + + $(SourceBuiltArtifactsPath) + + + $(SmokeTestsWarnOnLicenseScanDiffs) + + + $(SmokeTestsWarnOnSdkContentDiffs) + + + + + + + + $(TargetDir)packages/ + $(ArtifactsAssetsDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) + + + + + + + + + + + + + $(IntermediateOutputPath)prereq-packages/ + + + + + + + + + diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.sln b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.sln deleted file mode 100644 index 714ed63b6e67..000000000000 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.sln +++ /dev/null @@ -1,25 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.32112.339 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.SourceBuild.SmokeTests", "Microsoft.DotNet.SourceBuild.SmokeTests.csproj", "{FEB5A0B5-460B-432A-BED8-243557188AFF}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {FEB5A0B5-460B-432A-BED8-243557188AFF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FEB5A0B5-460B-432A-BED8-243557188AFF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FEB5A0B5-460B-432A-BED8-243557188AFF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FEB5A0B5-460B-432A-BED8-243557188AFF}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {75929D76-EEB5-4793-8335-DF623CC72B56} - EndGlobalSection -EndGlobal diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs index e140df73ba62..48198cbd9458 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs @@ -23,9 +23,11 @@ public class OmniSharpTests : SdkTests private string OmniSharpDirectory { get; } = Path.Combine(Directory.GetCurrentDirectory(), nameof(OmniSharpTests)); + public bool ShouldVerifyScenario => !Config.ExcludeOmniSharp && Config.TargetArchitecture != "ppc64le" && Config.TargetArchitecture != "s390x"; + public OmniSharpTests(ITestOutputHelper outputHelper) : base(outputHelper) { } - [SkippableTheory(Config.ExcludeOmniSharpEnv, skipOnTrueEnv: true, skipArchitectures: new[] { "ppc64le", "s390x" })] + [ConditionalTheoryAttribute(typeof(OmniSharpTests), nameof(ShouldVerifyScenario))] [InlineData(DotNetTemplate.BlazorWasm)] [InlineData(DotNetTemplate.ClassLib)] [InlineData(DotNetTemplate.Console)] diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/PoisonTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/PoisonTests.cs index 10a794fc412e..eccf5de267be 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/PoisonTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/PoisonTests.cs @@ -11,9 +11,11 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests { public class PoisonTests : SdkTests { + public bool ShouldRunPoisonTests => !string.IsNullOrWhiteSpace(Config.PoisonReportPath); + public PoisonTests(ITestOutputHelper outputHelper) : base(outputHelper) { } - [SkippableFact(Config.PoisonReportPathEnv, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(PoisonTests), nameof(ShouldRunPoisonTests))] public void VerifyUsage() { if (!File.Exists(Config.PoisonReportPath)) diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md index 9e9586075efc..7301a29e04c6 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md @@ -1,7 +1,22 @@ # Source Build Smoke Tests -* Run these tests via `build.sh --run-smoke-test` -* Various configuration settings are stored in `Config.cs` +* Run these tests via `build.sh --projects test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj --test` + +The following properties are automatically available during test execution but can be overwritten: +- MsftSdkTarballPath +- PoisonUsageReportFile +- SdkTarballPath +- SourceBuiltArtifactsPath + +Optional msbuild properties: +- SmokeTestsCustomSourceBuiltPackagesPath +- SmokeTestsExcludeOmniSharpTests +- SmokeTestsIncludeArtifactsSize +- SmokeTestsPrereqsPath +- SmokeTestsWarnOnLicenseScanDiffs +- SmokeTestsWarnOnSdkContentDiffs + +Make sure to rebuild the test project when changing one of those values. ## Dependencies @@ -15,5 +30,5 @@ The following programs are used by some tests: ## Prereq Packages Some prerelease scenarios, usually security updates, require non-source-built packages which are not publicly available. -Specify the directory where these packages can be found via the `SMOKE_TESTS_PREREQS_PATH` environment variable when running tests via `build.sh --run-smoke-test` e.g. -`SMOKE_TESTS_PREREQS_PATH=prereqs/packages/smoke-test-prereqs`. +Specify the directory where these packages can be found via the `SmokeTestsPrereqsPath` msbuild property when running tests via `build.sh --projects test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj --test` e.g. +`/p:SmokeTestsPrereqsPath=prereqs/packages/smoke-test-prereqs`. \ No newline at end of file diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs index 29bb7dbc55ca..47bbf542bb48 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs @@ -21,6 +21,7 @@ public class SdkContentTests : SdkTests { private const string MsftSdkType = "msft"; private const string SourceBuildSdkType = "sb"; + public bool ShouldCompareSdkContent => !string.IsNullOrWhiteSpace(Config.MsftSdkTarballPath) && !string.IsNullOrWhiteSpace(Config.SdkTarballPathEnv); public SdkContentTests(ITestOutputHelper outputHelper) : base(outputHelper) { } @@ -31,7 +32,7 @@ public SdkContentTests(ITestOutputHelper outputHelper) : base(outputHelper) { } /// This makes the baseline durable between releases. This does mean however, entries /// in the baseline may appear identical if the diff is version specific. /// - [SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.SdkTarballPathEnv }, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(SdkContentTests), nameof(ShouldCompareSdkContent))] public void CompareMsftToSbFileList() { const string msftFileListingFileName = "msftSdkFiles.txt"; @@ -44,7 +45,7 @@ public void CompareMsftToSbFileList() BaselineHelper.CompareBaselineContents("MsftToSbSdkFiles.diff", diff, OutputHelper, Config.WarnOnSdkContentDiffs); } - [SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.SdkTarballPathEnv }, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(SdkContentTests), nameof(ShouldCompareSdkContent))] public void CompareMsftToSbAssemblyVersions() { Assert.NotNull(Config.MsftSdkTarballPath); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableFactAttribute.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableFactAttribute.cs deleted file mode 100644 index c1314e19e8b5..000000000000 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableFactAttribute.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Linq; -using Xunit; - -namespace Microsoft.DotNet.SourceBuild.SmokeTests; - -/// -/// A Fact that will be skipped based on the specified environment variable's value. -/// -internal class SkippableFactAttribute : FactAttribute -{ - public SkippableFactAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) => - EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envName); - - public SkippableFactAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) => - EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envNames); - - public static void EvaluateSkips(bool skipOnNullOrWhiteSpaceEnv, bool skipOnTrueEnv, bool skipOnFalseEnv, string[] skipArchitectures, Action setSkip, params string[] envNames) - { - foreach (string envName in envNames) - { - string? envValue = Environment.GetEnvironmentVariable(envName); - - if (skipOnNullOrWhiteSpaceEnv && string.IsNullOrWhiteSpace(envValue)) - { - setSkip($"Skipping because `{envName}` is null or whitespace"); - break; - } - else if (skipOnTrueEnv && bool.TryParse(envValue, out bool boolValue) && boolValue) - { - setSkip($"Skipping because `{envName}` is set to True"); - break; - } - else if (skipOnFalseEnv && (!bool.TryParse(envValue, out boolValue) || !boolValue)) - { - setSkip($"Skipping because `{envName}` is set to False or an invalid value"); - break; - } - } - - if (skipArchitectures != null) { - string? arch = Config.TargetArchitecture; - if (skipArchitectures.Contains(arch)) - { - setSkip($"Skipping because arch is `{arch}`"); - } - } - } -} diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableTheoryAttribute.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableTheoryAttribute.cs deleted file mode 100644 index a43228c0d2a6..000000000000 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableTheoryAttribute.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Xunit; - -namespace Microsoft.DotNet.SourceBuild.SmokeTests; - -/// -/// A Theory that will be skipped based on the specified environment variable's value. -/// -internal class SkippableTheoryAttribute : TheoryAttribute -{ - public SkippableTheoryAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) => - SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envName); - - public SkippableTheoryAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) => - SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envNames); -} diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs index 7d6278b80b53..460004b8222d 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs @@ -15,9 +15,11 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests; public class SourceBuiltArtifactsTests : SdkTests { + public bool ShouldVerifyVersionFile => !string.IsNullOrWhiteSpace(Config.SourceBuiltArtifactsPath); + public SourceBuiltArtifactsTests(ITestOutputHelper outputHelper) : base(outputHelper) { } - [SkippableFact(Config.SourceBuiltArtifactsPathEnv, skipOnNullOrWhiteSpaceEnv: true)] + [ConditionalFact(typeof(SourceBuiltArtifactsTests), nameof(ShouldVerifyVersionFile))] public void VerifyVersionFile() { Assert.NotNull(Config.SourceBuiltArtifactsPath); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs index 228a65a61e94..e069e36206af 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs @@ -160,14 +160,6 @@ public static void LogWarningMessage(this ITestOutputHelper outputHelper, string outputHelper.WriteLine("##vso[task.complete result=SucceededWithIssues;]"); } - public static void ValidateNotNullOrWhiteSpace(string? variable, string variableName) - { - if (string.IsNullOrWhiteSpace(variable)) - { - throw new ArgumentException($"{variableName} is null, empty, or whitespace."); - } - } - public static string GetFile(string path, string pattern) { string[] files = Directory.GetFiles(path, pattern, SearchOption.AllDirectories); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Utilities.cs b/src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Utilities.cs index a56b4f0259ed..16b0da345804 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Utilities.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Utilities.cs @@ -141,86 +141,4 @@ public static IEnumerable GetTarballContentNames(string tarballPath) throw new InvalidOperationException($"Unsupported archive format: {tarballPath}"); } } - - public static void ExtractNupkg(string package, string outputDir) - { - Directory.CreateDirectory(outputDir); - - using ZipArchive zip = ZipFile.OpenRead(package); - foreach (ZipArchiveEntry entry in zip.Entries) - { - string outputPath = Path.Combine(outputDir, entry.FullName); - Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); - entry.ExtractToFile(outputPath); - } - } - - public static async Task RetryAsync(Func executor, ITestOutputHelper outputHelper) - { - await Utilities.RetryAsync( - async () => - { - try - { - await executor(); - return null; - } - catch (Exception e) - { - return e; - } - }, - outputHelper); - } - - private static async Task RetryAsync(Func> executor, ITestOutputHelper outputHelper) - { - const int maxRetries = 5; - const int waitFactor = 5; - - int retryCount = 0; - - Exception? exception = await executor(); - while (exception != null) - { - retryCount++; - if (retryCount >= maxRetries) - { - throw new InvalidOperationException($"Failed after {retryCount} retries.", exception); - } - - int waitTime = Convert.ToInt32(Math.Pow(waitFactor, retryCount - 1)); - if (outputHelper != null) - { - outputHelper.WriteLine($"Retry {retryCount}/{maxRetries}, retrying in {waitTime} seconds..."); - } - - Thread.Sleep(TimeSpan.FromSeconds(waitTime)); - exception = await executor(); - } - } - - public static void LogWarningMessage(this ITestOutputHelper outputHelper, string message) - { - string prefix = "##vso[task.logissue type=warning;]"; - - outputHelper.WriteLine($"{Environment.NewLine}{prefix}{message}.{Environment.NewLine}"); - outputHelper.WriteLine("##vso[task.complete result=SucceededWithIssues;]"); - } - - public static void ValidateNotNullOrWhiteSpace(string? variable, string variableName) - { - if (string.IsNullOrWhiteSpace(variable)) - { - throw new ArgumentException($"{variableName} is null, empty, or whitespace."); - } - } - - public static string GetFile(string path, string pattern) - { - string[] files = Directory.GetFiles(path, pattern, SearchOption.AllDirectories); - Assert.False(files.Length > 1, $"Found multiple files matching the pattern {pattern}: {Environment.NewLine}{string.Join(Environment.NewLine, files)}"); - Assert.False(files.Length == 0, $"Did not find any files matching the pattern {pattern}"); - return files[0]; - } }