diff --git a/eng/testing/WasmRunnerTemplate.cmd b/eng/testing/WasmRunnerTemplate.cmd index a668ce2c6a464..026a06066c318 100644 --- a/eng/testing/WasmRunnerTemplate.cmd +++ b/eng/testing/WasmRunnerTemplate.cmd @@ -57,6 +57,10 @@ if [%XHARNESS_ARGS%] == [] ( set "XHARNESS_ARGS=%JS_ENGINE% %JS_ENGINE_ARGS% %BROWSER_PATH% %MAIN_JS%" ) +if [%XUNIT_RANDOM_ORDER_SEED%] NEQ [] ( + set "WasmXHarnessMonoArgs=%WasmXHarnessMonoArgs% --setenv=XUNIT_RANDOM_ORDER_SEED=%XUNIT_RANDOM_ORDER_SEED%" +) + echo EXECUTION_DIR=%EXECUTION_DIR% echo SCENARIO=%SCENARIO% echo XHARNESS_OUT=%XHARNESS_OUT% diff --git a/eng/testing/WasmRunnerTemplate.sh b/eng/testing/WasmRunnerTemplate.sh index ae78d5304e8ae..6c054ba9d23e1 100644 --- a/eng/testing/WasmRunnerTemplate.sh +++ b/eng/testing/WasmRunnerTemplate.sh @@ -53,6 +53,10 @@ if [[ -z "$XHARNESS_ARGS" ]]; then XHARNESS_ARGS="$JS_ENGINE $JS_ENGINE_ARGS $MAIN_JS" fi +if [[ -n "$XUNIT_RANDOM_ORDER_SEED" ]]; then + WasmXHarnessMonoArgs="${WasmXHarnessMonoArgs} --setenv=XUNIT_RANDOM_ORDER_SEED=${XUNIT_RANDOM_ORDER_SEED}" +fi + echo EXECUTION_DIR=$EXECUTION_DIR echo SCENARIO=$SCENARIO echo XHARNESS_OUT=$XHARNESS_OUT @@ -64,7 +68,6 @@ echo JS_ENGINE=$JS_ENGINE echo JS_ENGINE_ARGS=$JS_ENGINE_ARGS echo XHARNESS_ARGS=$XHARNESS_ARGS - pushd $EXECUTION_DIR # ========================= BEGIN Test Execution ============================= @@ -83,4 +86,4 @@ echo ----- end $(date) ----- exit code $_exitCode ------------------------------ echo "XHarness artifacts: $XHARNESS_OUT" -exit $_exitCode \ No newline at end of file +exit $_exitCode diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index da8da1d2f4bfa..837dc9716536a 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -51,6 +51,10 @@ $(AdditionalXHarnessArguments) -- -c=$(XUnitClassName) + + + + diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index 4e3addd55acb2..6b3a1d61d7eae 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -15,6 +15,7 @@ <_ShellCommandSeparator Condition="'$(OS)' != 'Windows_NT'">&& false <_WasmMainJSFileName Condition="'$(WasmMainJSPath)' != ''">$([System.IO.Path]::GetFileName('$(WasmMainJSPath)')) + true diff --git a/src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs b/src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs new file mode 100644 index 0000000000000..effe3a08e1404 --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading; +using Xunit.Abstractions; +using Xunit.Sdk; + +#nullable enable + +namespace TestUtilities; + +// Based on https://github.com/xunit/xunit/blob/v2/src/xunit.execution/Sdk/DefaultTestCaseOrderer.cs + +public class RandomTestCaseOrderer : ITestCaseOrderer +{ + public const string RandomSeedEnvironmentVariableName = "XUNIT_RANDOM_ORDER_SEED"; + + public static readonly Lazy LazySeed = new (GetSeed, LazyThreadSafetyMode.ExecutionAndPublication); + private readonly IMessageSink _diagnosticMessageSink; + + private static int GetSeed() + { + string? seedEnvVar = Environment.GetEnvironmentVariable(RandomSeedEnvironmentVariableName); + if (string.IsNullOrEmpty(seedEnvVar) || !int.TryParse(seedEnvVar, out int seed)) + { + seed = new Random().Next(); + } + + return seed; + } + + public RandomTestCaseOrderer(IMessageSink diagnosticMessageSink) + { + diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Using random seed for test cases: {LazySeed.Value}")); + _diagnosticMessageSink = diagnosticMessageSink; + } + + public IEnumerable OrderTestCases(IEnumerable testCases) where TTestCase : ITestCase + => TryRandomize(testCases.ToList(), _diagnosticMessageSink, out List? randomizedTests) + ? randomizedTests + : testCases; + + public static bool TryRandomize(List tests, IMessageSink messageSink, [NotNullWhen(true)] out List? randomizedTests) + { + randomizedTests = null; + try + { + randomizedTests = Randomize(tests.ToList()); + return true; + } + catch (Exception ex) + { + messageSink.OnMessage(new DiagnosticMessage($"Failed to randomize test cases: {ex}")); + return false; + } + + static List Randomize(List tests) + { + var result = new List(tests.Count); + + var randomizer = new Random(LazySeed.Value); + + while (tests.Count > 0) + { + int next = randomizer.Next(tests.Count); + result.Add(tests[next]); + tests.RemoveAt(next); + } + + return result; + } + } +} diff --git a/src/libraries/Common/tests/TestUtilities/RandomTestCollectionOrderer.cs b/src/libraries/Common/tests/TestUtilities/RandomTestCollectionOrderer.cs new file mode 100644 index 0000000000000..758fef8039755 --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/RandomTestCollectionOrderer.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +#nullable enable + +namespace TestUtilities; + +public class RandomTestCollectionOrderer : ITestCollectionOrderer +{ + private readonly IMessageSink _diagnosticMessageSink; + + public RandomTestCollectionOrderer(IMessageSink diagnosticMessageSink) + { + diagnosticMessageSink.OnMessage(new DiagnosticMessage( + $"Using random seed for collections: {RandomTestCaseOrderer.LazySeed.Value}")); + _diagnosticMessageSink = diagnosticMessageSink; + } + + public IEnumerable OrderTestCollections(IEnumerable testCollections) + => RandomTestCaseOrderer.TryRandomize(testCollections.ToList(), _diagnosticMessageSink, out List? randomizedTests) + ? randomizedTests + : testCollections; +} diff --git a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj index ecbb20d8ee57c..b24afba9d9e70 100644 --- a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj +++ b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj @@ -43,6 +43,9 @@ variant from the Common folder and adding the missing members manually. --> + + + diff --git a/src/libraries/Common/tests/Tests/RandomizedTestOrderAssemblyInfo.cs b/src/libraries/Common/tests/Tests/RandomizedTestOrderAssemblyInfo.cs new file mode 100644 index 0000000000000..3b051bad65bc8 --- /dev/null +++ b/src/libraries/Common/tests/Tests/RandomizedTestOrderAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +[assembly: TestCaseOrderer("TestUtilities.RandomTestCaseOrderer", "TestUtilities")] +[assembly: TestCollectionOrderer("TestUtilities.RandomTestCollectionOrderer", "TestUtilities")] diff --git a/src/libraries/System.Private.Xml/tests/Writers/XmlWriterApi/System.Xml.RW.XmlWriterApi.Tests.csproj b/src/libraries/System.Private.Xml/tests/Writers/XmlWriterApi/System.Xml.RW.XmlWriterApi.Tests.csproj index 9569073b4185a..7ac73f30358cf 100644 --- a/src/libraries/System.Private.Xml/tests/Writers/XmlWriterApi/System.Xml.RW.XmlWriterApi.Tests.csproj +++ b/src/libraries/System.Private.Xml/tests/Writers/XmlWriterApi/System.Xml.RW.XmlWriterApi.Tests.csproj @@ -1,6 +1,7 @@ $(NetCoreAppCurrent) + false diff --git a/src/libraries/System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj b/src/libraries/System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj index e21bc7e673f5f..de19f64769380 100644 --- a/src/libraries/System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj +++ b/src/libraries/System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj @@ -8,6 +8,9 @@ false --setenv=DOTNET_MODIFIABLE_ASSEMBLIES=debug + + + false diff --git a/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj b/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj index 5dd112190af3c..976ae7909313e 100644 --- a/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj +++ b/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj @@ -2,6 +2,9 @@ true $(NetCoreAppCurrent) + + + false @@ -51,4 +54,4 @@ - \ No newline at end of file +