diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd new file mode 100644 index 00000000000..13f080df302 --- /dev/null +++ b/tests/CoreFX/build-and-run-test.cmd @@ -0,0 +1,54 @@ +:: Test build and execution wrapper for CoreFX tests +:: +:: This wrapper is called for each of CoreFX's tests by runtest.cmd +:: +:: %1 contains test folder +:: %2 contains test exe name +:: +@echo OFF +setlocal ENABLEDELAYEDEXPANSION + +set TestFolder=%1 + +:: +:: We're only interested in referencing the xunit runner - the test dlls will be imported by the test wrapper project +:: +set TestExecutable=xunit.console.netcore +set TestFileName=%2 + + +:: Copy the artefacts we need to compile and run the xunit exe +copy /Y "%~dp0\runtest\CoreFXTestHarness\*" "%TestFolder%" >nul + +:: Create log dir if it doesn't exist +if not exist %XunitLogDir% md %XunitLogDir% + +if not exist %TestFolder%\%TestExecutable%.exe ( + :: Not a test we support, exit silently + exit /b 0 +) + +:: Workaround until we have a better reflection engine +:: Add name of currently executing test to rd.xml +powershell -Command "(Get-Content %TestFolder%\default.rd.xml).replace('*Application*', '%TestFileName%') | Set-Content %TestFolder%\default.rd.xml" + +if "%CoreRT_BuildArch%" == "x64" ( + call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" >nul +) + +echo Building %TestFileName% + +call "%CoreRT_CliDir%\dotnet.exe" publish %TestFolder%\Test.csproj /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:TestRootDir=%~dp0 /p:OSGroup=%CoreRT_BuildOS% /p:ExecutableName=%TestExecutable% /nologo +if errorlevel 1 ( + echo Building %TestFileName% failed + exit /b 1 +) + +echo Executing %TestFileName% - writing logs to %XunitLogDir%\%TestFileName%.xml + +if not exist "%TestFolder%\native\%TestExecutable%".exe ( + echo ERROR:Native binary not found Unable to run test. + exit /b 1 +) + +call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -xml %XunitLogDir%\%TestFileName%.xml \ No newline at end of file diff --git a/tests/CoreFX/corerun b/tests/CoreFX/corerun new file mode 100755 index 00000000000..76464611934 --- /dev/null +++ b/tests/CoreFX/corerun @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# This is the Unix equivalent of build-and-run-test.cmd +# It is invoked by each test's bash script. The reason it's called corerun is that +# the unix CoreCLR tests don't have a custom runner override environment variable. +# See issue https://github.com/dotnet/coreclr/issues/9007 +# The CoreFX alternative is named corerun to keep parity with the CoreCLR testing convention + + +export TestFolderName=$1 +export TestFileName=$(basename ${TestFolderName}) +export TestRootDir=${CoreRT_TestRoot}/CoreFX +export LogDir=$2 +# +# We're only interested in referencing the xunit runner - the test dlls will be imported by the test wrapper project +# +TestExecutable=xunit.console.netcore + +source "$CoreRT_TestRoot/coredump_handling.sh" + +if [[ $CoreRT_EnableCoreDumps == 1 ]]; then + set_up_core_dump_generation +fi + +if [ ! -e "${TestFolderName}/${TestExecutable}.exe" ]; then + echo "not found in ${TestFolderName}/${TestExecutable}.exe" + # Not a test we support, exit silently + exit 0 +fi + +# Copy the artefacts we need to compile and run the xunit exe +cp -a "${CoreRT_TestRoot}/CoreFX/runtest/CoreFXTestHarness/." "${TestFolderName}" +export __exitcode=$? +if [ ${__exitcode} != 0 ]; +then + exit ${__exitcode} +fi + +# Workaround until we have a better reflection engine +# Add name of currently executing test to rd.xml +sed -i.bak "s/\*Application\*/${TestFileName}/g" "${TestFolderName}/default.rd.xml" + +echo Building ${TestFileName} + +${CoreRT_CliBinDir}/dotnet publish "/p:IlcPath=${CoreRT_ToolchainDir}" "/p:DebugSymbols=false" /p:OSGroup=${CoreRT_BuildOS} "/p:Configuration=${CoreRT_BuildType}" "/p:FrameworkLibPath=${CoreRT_TestRoot}/../bin/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/lib" "/p:FrameworkObjPath=${CoreRT_TestRoot}/../bin/obj/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/Framework" /p:DisableFrameworkLibGeneration=true /p:TestRootDir=${TestRootDir} /p:ExecutableName=${TestExecutable} "/p:OutputPath=/${TestFolderName}/" ${TestFolderName}/Test.csproj +export __exitcode=$? +if [ ${__exitcode} != 0 ]; +then + exit ${__exitcode} +fi + +echo Executing ${TestFileName} - writing logs to ${LogDir}/${TestFileName}.xml +chmod +x ${TestFolderName}/publish/${TestExecutable} +${TestFolderName}/publish/${TestExecutable} ${TestFolderName}/${TestFileName}.dll -xml ${LogDir}/${TestFileName}.xml +export __exitcode=$? + +exit ${__exitcode} diff --git a/tests/CoreFX/dependencies.props b/tests/CoreFX/dependencies.props new file mode 100644 index 00000000000..28a73b4ab9f --- /dev/null +++ b/tests/CoreFX/dependencies.props @@ -0,0 +1,17 @@ + + + + + + 0.1.0-e160909-1 + 10.0.1 + 2.3.0-beta1-build3642 + + 2.2.0-beta2-build3300 + 2.0.1 + 2.1.0-preview2-02516-02 + 4.5.0-preview2-26219-0 + + diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj new file mode 100644 index 00000000000..91adc31c9bb --- /dev/null +++ b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj @@ -0,0 +1,73 @@ + + + + + + + + $(ExecutableName) + .exe + Exe + + + + $(FrameworkLibPath)\..\tools\ + netcoreapp2.0 + + $(NoWarn);NU1603 + + + + + + + + $(XunitAbstractionsVersion) + + + $(XunitPackageVersion) + + + $(XunitPackageVersion) + + + $(XunitPackageVersion) + + + $(XunitRunnerUtilityVersion) + + + $(XunitPackageVersion) + + + $(XunitNetcoreExtensionsVersion) + + + $(CoreFxTestUtilitiesVersion) + + + + + + + + + + + $(MSBuildProjectDirectory)\ + $(MSBuildProjectDirectory)\ + + + + + win-x64 + linux-x64 + osx-x64 + + + + + + + + diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml new file mode 100644 index 00000000000..81d5bc9e0a6 --- /dev/null +++ b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/CoreFX/runtest/dir.props b/tests/CoreFX/runtest/dir.props new file mode 100644 index 00000000000..e947c5a9968 --- /dev/null +++ b/tests/CoreFX/runtest/dir.props @@ -0,0 +1,33 @@ + + + + + + + $(OS) + + + + + false + $(MSBuildThisFileDirectory)\ + $(ProjectDir)src\ + $(ProjectDir)..\..\..\packages\ + $(ProjectDir)..\Tools\ + $(ToolsDir)dotnetcli/ + true + + + + + + + + + + + + + diff --git a/tests/CoreFX/runtest/runtest.cmd b/tests/CoreFX/runtest/runtest.cmd new file mode 100644 index 00000000000..33a3001865e --- /dev/null +++ b/tests/CoreFX/runtest/runtest.cmd @@ -0,0 +1,100 @@ +@if not defined __echo @echo off +setlocal EnableDelayedExpansion + +:: Set the default arguments +set __BuildArch=x64 +set __BuildType=Debug +set __BuildOS=Windows_NT + +:: Default to highest Visual Studio version available +set __VSVersion=vs2017 +set __VSProductVersion=150 + +:: Define a prefix for most output progress messages that come from this script. That makes +:: it easier to see where these are coming from. Note that there is a trailing space here. +set __MsgPrefix=RUNTEST: + +set __ProjectDir=%~dp0 +:: remove trailing slash +if %__ProjectDir:~-1%==\ set "__ProjectDir=%__ProjectDir:~0,-1%" +set "__ProjectFilesDir=%__ProjectDir%" +set "__RootBinDir=%__ProjectDir%\..\bin" +set "__LogsDir=%__RootBinDir%\Logs" + +:Arg_Loop +if "%1" == "" goto ArgsDone + +if /i "%1" == "/?" goto Usage +if /i "%1" == "-?" goto Usage +if /i "%1" == "/h" goto Usage +if /i "%1" == "-h" goto Usage +if /i "%1" == "/help" goto Usage +if /i "%1" == "-help" goto Usage + +if /i "%1" == "x64" (set __BuildArch=x64&set __MSBuildBuildArch=x64&shift&goto Arg_Loop) +if /i "%1" == "x86" (set __BuildArch=x86&set __MSBuildBuildArch=x86&shift&goto Arg_Loop) + +if /i "%1" == "debug" (set __BuildType=Debug&shift&goto Arg_Loop) +if /i "%1" == "release" (set __BuildType=Release&shift&goto Arg_Loop) + +if /i "%1" == "LogsDir" (set __LogsDir=%2&shift&shift&goto Arg_Loop) + +if /i not "%1" == "msbuildargs" goto SkipMsbuildArgs + +set CORE_ROOT=%1 +echo %__MsgPrefix%CORE_ROOT is initially set to: "%CORE_ROOT%" +shift +:ArgsDone + +set "__TestWorkingDir=%CoreRT_TestRoot\CoreFX%" + +if not defined XunitTestBinBase set XunitTestBinBase=%__TestWorkingDir% + +if not exist %__LogsDir% md %__LogsDir% + +:: Check presence of VS +if not defined VS%__VSProductVersion%COMNTOOLS goto NoVS + +set __VSToolsRoot=!VS%__VSProductVersion%COMNTOOLS! +if %__VSToolsRoot:~-1%==\ set "__VSToolsRoot=%__VSToolsRoot:~0,-1%" + +:: Set the environment for the build- VS cmd prompt +echo %__MsgPrefix%Using environment: "%__VSToolsRoot%\VsDevCmd.bat" +call "%__VSToolsRoot%\VsDevCmd.bat" + +if not defined VSINSTALLDIR ( + echo %__MsgPrefix%Error: runtest.cmd should be run from a Visual Studio Command Prompt. Please see https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/developer-guide.md for build instructions. + exit /b 1 +) + +if not defined FXCustomTestLauncher ( + echo The CoreFX test launcher is not defined + exit /b 1 +) + +:: Iterate through unzipped CoreFX tests +for /D %%i in ("%XunitTestBinBase%\*" ) do ( + set TestFolderName=%%i + set TestFileName=%%~nxi + + echo %FXCustomTestLauncher% !TestFolderName! !TestFileName! + call %FXCustomTestLauncher% !TestFolderName! !TestFileName! +) + +exit /b 0 + +:Usage +echo. +echo Usage: +echo %0 BuildArch BuildType +echo where: +echo. +echo./? -? /h -h /help -help: view this message. +echo BuildArch- Optional parameter - x64 or x86 ^(default: x64^). +echo BuildType- Optional parameter - Debug, Release, or Checked ^(default: Debug^). +exit /b 1 + +:NoVS +echo Visual Studio 2017 ^(Community is free^) is a prerequisite to build this repository. +echo See: https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/developer-guide.md#prerequisites +exit /b 1 diff --git a/tests/CoreFX/runtest/runtest.sh b/tests/CoreFX/runtest/runtest.sh new file mode 100755 index 00000000000..c45535db150 --- /dev/null +++ b/tests/CoreFX/runtest/runtest.sh @@ -0,0 +1,71 @@ + +function print_usage { + echo '' + echo 'CoreFX test runner script.' + echo '' + echo 'Typical command line:' + echo '' + echo 'corefx/tests/runtest.sh' + echo ' --testRootDir="tests_downloaded/CoreFX"' + echo ' --testLauncher="tests/CoreFX/corerun"' + echo '' + echo 'Required arguments:' + echo ' --testRootDir= : Root directory of the CoreFX test build ' + echo ' --testLauncher= : Path to the test launcher script' +} + +function run_tests_in_directory { + local testDir=$1 + for testSubDir in ${testDir}/* ; do + # Build and run each test + echo Building ${testSubDir} + ${FXCustomTestLauncher} ${testSubDir} ${__LogDir} + done + +} + +# Exit code constants +readonly EXIT_CODE_SUCCESS=0 # Script ran normally. +readonly EXIT_CODE_EXCEPTION=1 # Script exited because something exceptional happened (e.g. bad arguments, Ctrl-C interrupt). +readonly EXIT_CODE_TEST_FAILURE=2 # Script completed successfully, but one or more tests failed. + +# Argument variables +testRootDir= +__LogDir= +FXCustomTestLauncher= + +for i in "$@" +do + case $i in + --testRootDir=*) + testRootDir=${i#*=} + ;; + --logdir=*) + __LogDir=${i#*=} + ;; + --testLauncher=*) + FXCustomTestLauncher=${i#*=} + ;; + *) + echo "Unknown switch: $i" + print_usage + exit $EXIT_CODE_SUCCESS + ;; + esac +done + +if [ -z "$testRootDir" ]; then + echo "--testRootDir is required." + print_usage + exit $EXIT_CODE_EXCEPTION +fi +if [ ! -d "$testRootDir" ]; then + echo "Directory specified by --testRootDir does not exist: $testRootDir" + exit $EXIT_CODE_EXCEPTION +fi + +if [ -z "$__LogDir" ]; then + __LogDir=$testRootDir +fi + +run_tests_in_directory ${testRootDir} diff --git a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj new file mode 100644 index 00000000000..3653a481442 --- /dev/null +++ b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj @@ -0,0 +1,15 @@ + + + + netcoreapp2.0 + Exe + + + + $(NewtonsoftJsonVersion) + + + $(SystemCommandLineVersion) + + + diff --git a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs new file mode 100644 index 00000000000..a063ad4e971 --- /dev/null +++ b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs @@ -0,0 +1,251 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Net.Http; +using System.Threading.Tasks; +using Newtonsoft.Json; + +namespace CoreFX.TestUtils.TestFileSetup +{ + /// + /// Defines the set of flags that represent exit codes + /// + [Flags] + public enum ExitCode : int + { + Success = 0, + HttpError = 1, + IOError = 2, + UnknownError = 10 + + } + + /// + /// This helper class is used to fetch CoreFX tests from a specified URL, unarchive them and create a flat directory structure + /// through which to iterate. + /// + public static class TestFileSetup + { + private static HttpClient httpClient; + private static bool cleanTestBuild = false; + + private static string outputDir; + private static string testUrl; + private static string testListPath; + + public static void Main(string[] args) + { + ExitCode exitCode = ExitCode.UnknownError; + ArgumentSyntax argSyntax = ParseCommandLine(args); + + if (!Directory.Exists(outputDir)) + { + try + { + Directory.CreateDirectory(outputDir); + } + catch (IOException) + { + exitCode = ExitCode.IOError; + Environment.Exit((int)exitCode); + } + } + + // parse args + try + { + SetupTests(testUrl, outputDir, ReadTestNames(testListPath)).Wait(); + exitCode = ExitCode.Success; + + } + catch (HttpRequestException) + { + exitCode = ExitCode.HttpError; + } + catch (IOException) + { + exitCode = ExitCode.IOError; + } + + Environment.Exit((int)exitCode); + } + + public static ArgumentSyntax ParseCommandLine(string[] args) + { + ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax => + { + syntax.DefineOption("out|outDir|outputDirectory", ref outputDir, "Directory where tests are downloaded"); + syntax.DefineOption("testUrl", ref testUrl, "URL, pointing to the list of tests"); + syntax.DefineOption("testListJsonPath", ref testListPath, "JSON-formatted list of test assembly names to download"); + syntax.DefineOption("clean", ref cleanTestBuild, "Remove all previously built test assemblies"); + }); + + return argSyntax; + } + + public static Dictionary ReadTestNames(string testFilePath) + { + Debug.Assert(File.Exists(testFilePath)); + + Dictionary testNames = new Dictionary(); + + // We're being a passed a list of test assembly names, so anything that's not a string is invalid + using (var sr = new StreamReader(testFilePath)) + using (var jsonReader = new JsonTextReader(sr)) + { + while (jsonReader.Read()) + { + if (jsonReader.TokenType == JsonToken.String) + { + testNames.Add(jsonReader.Value.ToString(), string.Empty); + } + } + } + + return testNames; + } + + public static async Task SetupTests(string jsonUrl, string destinationDirectory, Dictionary testNames = null, bool runAllTests = false) + { + Debug.Assert(Directory.Exists(destinationDirectory)); + Debug.Assert(runAllTests || testNames != null); + + string tempDirPath = Path.Combine(destinationDirectory, "temp"); + if (!Directory.Exists(tempDirPath)) + { + Directory.CreateDirectory(tempDirPath); + } + Dictionary testPayloads = await GetTestUrls(jsonUrl, testNames, runAllTests); + + await GetTestArchives(testPayloads, tempDirPath); + ExpandArchivesInDirectory(tempDirPath, destinationDirectory); + + Directory.Delete(tempDirPath); + } + public static async Task> GetTestUrls(string jsonUrl, Dictionary testNames = null, bool runAllTests = false) + { + if (httpClient is null) + { + httpClient = new HttpClient(); + } + + Debug.Assert(runAllTests || testNames != null); + + // Set up the json stream reader + using (var responseStream = await httpClient.GetStreamAsync(jsonUrl)) + using (var streamReader = new StreamReader(responseStream)) + using (var jsonReader = new JsonTextReader(streamReader)) + { + // Manual parsing - we only need to key-value pairs from each object and this avoids deserializing all of the work items into objects + string markedTestName = string.Empty; + string currentPropertyName = string.Empty; + + while (jsonReader.Read()) + { + if (jsonReader.Value != null) + { + switch (jsonReader.TokenType) + { + case JsonToken.PropertyName: + currentPropertyName = jsonReader.Value.ToString(); + break; + case JsonToken.String: + if (currentPropertyName.Equals("WorkItemId")) + { + string currentTestName = jsonReader.Value.ToString(); + + if (runAllTests || testNames.ContainsKey(currentTestName)) + { + markedTestName = currentTestName; + } + } + else if (currentPropertyName.Equals("PayloadUri") && markedTestName != string.Empty) + { + testNames[markedTestName] = jsonReader.Value.ToString(); + markedTestName = string.Empty; + } + break; + } + } + } + + } + return testNames; + } + + public static async Task GetTestArchives(Dictionary testPayloads, string downloadDir) + { + if (httpClient is null) + { + httpClient = new HttpClient(); + } + + foreach (string testName in testPayloads.Keys) + { + string payloadUri = testPayloads[testName]; + + if (!Uri.IsWellFormedUriString(payloadUri, UriKind.Absolute)) + continue; + + using (var response = await httpClient.GetStreamAsync(payloadUri)) + { + if (response.CanRead) + { + // Create the test setup directory if it doesn't exist + if (!Directory.Exists(downloadDir)) + { + Directory.CreateDirectory(downloadDir); + } + + // CoreFX test archives are output as .zip regardless of platform + string archivePath = Path.Combine(downloadDir, testName + ".zip"); + + // Copy to a temp folder + using (FileStream file = new FileStream(archivePath, FileMode.Create)) + { + await response.CopyToAsync(file); + } + + } + } + } + } + + public static void ExpandArchivesInDirectory(string archiveDirectory, string destinationDirectory, bool cleanup = true) + { + Debug.Assert(Directory.Exists(archiveDirectory)); + Debug.Assert(Directory.Exists(destinationDirectory)); + + string[] archives = Directory.GetFiles(archiveDirectory, "*.zip", SearchOption.TopDirectoryOnly); + + foreach (string archivePath in archives) + { + string destinationDirName = Path.Combine(destinationDirectory, Path.GetFileNameWithoutExtension(archivePath)); + + // If doing clean test build - delete existing artefacts + if (Directory.Exists(destinationDirName) && cleanTestBuild) + { + Directory.Delete(destinationDirName, true); + } + + ZipFile.ExtractToDirectory(archivePath, destinationDirName); + + + // Delete archives + if (cleanup) + { + File.Delete(archivePath); + } + } + } + + + } +} diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj b/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj new file mode 100644 index 00000000000..3653a481442 --- /dev/null +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj @@ -0,0 +1,15 @@ + + + + netcoreapp2.0 + Exe + + + + $(NewtonsoftJsonVersion) + + + $(SystemCommandLineVersion) + + + diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs new file mode 100644 index 00000000000..3c111699024 --- /dev/null +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs @@ -0,0 +1,115 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +/// +/// This helper class reads xunit xml log files and prints the tallied test results to the console +/// +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Net.Http; +using System.Reflection; +using System.Threading.Tasks; +using System.Xml; +using Newtonsoft.Json; + +namespace CoreFX.TestUtils.XUnit +{ + public static class ResultFormatter + { + private static string logDir; + private static string logPattern; + + public static void Main(string[] args) + { + ArgumentSyntax syntax = ParseCommandLine(args); + IEnumerable logFiles = DiscoverLogs(logDir, logPattern); + PrintTotals(logFiles); + } + + private static void PrintTotals(IEnumerable logFiles) + { + int total = 0; + int passed = 0; + int failed = 0; + int skipped = 0; + ulong timeElapsed = 0; + + foreach(string logFile in logFiles) + { + // XMLReader escapes the character sequence \\.. as just a single backslash \ - Is this intended behavior? + using (XmlReader reader = XmlReader.Create(logFile.Replace(@"\\..", @"\.."))) + { + reader.MoveToContent(); + reader.ReadToDescendant("collection"); + do + { + // Get total tests in current element + string totalAttr = reader.GetAttribute("total"); + int currentTotal; + Int32.TryParse(totalAttr, out currentTotal); + total += currentTotal; + + // Get passed tests + string passedAttr = reader.GetAttribute("passed"); + int currentPassed; + Int32.TryParse(passedAttr, out currentPassed); + passed += currentPassed; + + // Get failed tests + string failedAttr = reader.GetAttribute("failed"); + int currentFailed; + Int32.TryParse(failedAttr, out currentFailed); + failed += currentFailed; + + // Get skipped tests + string skippedAttr = reader.GetAttribute("skipped"); + int currentSkipped; + Int32.TryParse(skippedAttr, out currentSkipped); + skipped += currentSkipped; + + // Get time elapsed + string timeAttr = reader.GetAttribute("time"); + ulong currentTime; + UInt64.TryParse(timeAttr, out currentTime); + timeElapsed += currentTime; + + } while (reader.ReadToNextSibling("collection")); + + } + } + + Console.WriteLine("=== CoreFX TEST EXECUTION SUMMARY ===: "); + Console.WriteLine(String.Format("Total: {0}, Passed: {1}, Failed: {2}, Skipped: {3}", total, passed, failed, timeElapsed)); + Console.WriteLine("Detailed logs written to: " + logDir); + + } + + private static ArgumentSyntax ParseCommandLine(string[] args) + { + + ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax => + { + syntax.DefineOption("log|logDirectory|logDir", ref logDir, "Path to directory of xml test results"); + syntax.DefineOption("pattern|p", ref logPattern, "Pattern of XUnit log filenames for which to search"); + }); + + return argSyntax; + } + + public static IEnumerable DiscoverLogs(string logDirectory, string logPattern) + { + Debug.Assert(Directory.Exists(logDirectory)); + Console.WriteLine(logDirectory); + var logFiles = Directory.EnumerateFiles(logDirectory, logPattern, SearchOption.AllDirectories); + + return logFiles; + } + + + } +} diff --git a/tests/CoreFXTestListURL.txt b/tests/CoreFXTestListURL.txt new file mode 100644 index 00000000000..b7900747e40 --- /dev/null +++ b/tests/CoreFXTestListURL.txt @@ -0,0 +1 @@ +https://dotnetbuilddrops.blob.core.windows.net/build-987f9e019ca4405c9127f02516782d07/TestList.json?sv=2015-04-05&sr=c&sig=ETMKT2IFbiyndcPMJtEMeBwySuZURBxQ2eqlJ4xVXi4%3D&st=2018-03-13T21%3A14%3A10.2227157Z&se=2118-02-17T21%3A14%3A10.2242145Z&sp=r \ No newline at end of file diff --git a/tests/TopN.CoreFX.issues.json b/tests/TopN.CoreFX.issues.json new file mode 100644 index 00000000000..5ebc82f4012 --- /dev/null +++ b/tests/TopN.CoreFX.issues.json @@ -0,0 +1,3 @@ +[ + "System.Collections.Tests", +] diff --git a/tests/runtest.cmd b/tests/runtest.cmd index 7550edc94a0..d75fe237e10 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -10,6 +10,7 @@ set CoreRT_BuildOS=Windows_NT set CoreRT_TestRun=true set CoreRT_TestCompileMode= set CoreRT_RunCoreCLRTests= +set CoreRT_RunCoreFXTests= set CoreRT_CoreCLRTargetsFile= set CoreRT_TestLogFileName=testresults.xml set CoreRT_TestName=* @@ -48,6 +49,7 @@ if /i "%1" == "/coreclr" ( :ExtRepoTestsOk goto ArgLoop ) +if /i "%1" == "/corefx" (set CoreRT_RunCoreFXTests=true&shift&goto ArgLoop) if /i "%1" == "/coreclrsingletest" (set CoreRT_RunCoreCLRTests=true&set CoreRT_CoreCLRTest=%2&shift&shift&goto ArgLoop) if /i "%1" == "/mode" (set CoreRT_TestCompileMode=%2&shift&shift&goto ArgLoop) if /i "%1" == "/test" (set CoreRT_TestName=%2&shift&shift&goto ArgLoop) @@ -67,6 +69,7 @@ echo /mode : Optionally restrict to a single code generator. Specify echo /test : Run a single test by folder name (ie, BasicThreading) echo /runtest : Should just compile or run compiled binary? Specify: true/false. Default: true. echo /coreclr : Download and run the CoreCLR repo tests +echo /corefx : Download and run the CoreFX repo tests echo /coreclrsingletest ^ echo : Run a single CoreCLR repo test echo /multimodule : Compile the framework as a .lib and link tests against it (only supports ryujit) @@ -121,7 +124,8 @@ call "%_VSCOMNTOOLS%\VsDevCmd.bat" :RunVCVars call "!VS150COMNTOOLS!\..\..\VC\Auxiliary\Build\vcvarsall.bat" %CoreRT_HostArch% -if "%CoreRT_RunCoreCLRTests%"=="true" goto :TestExtRepo +if "%CoreRT_RunCoreCLRTests%"=="true" goto :TestExtRepoCoreCLR +if "%CoreRT_RunCoreFXTests%"=="true" goto :TestExtRepoCoreFX if /i "%__BuildType%"=="Debug" ( set __LinkLibs=msvcrtd.lib @@ -340,9 +344,52 @@ goto :eof powershell -Command Write-Host %1 -foreground "red" exit /b -1 +:RestoreCoreFXTests + + :: Explicitly restore the test helper project + "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "%CoreRT_TestFileHelperProjectPath%" + if errorlevel 1 ( + exit /b 1 + ) + "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "%CoreRT_XunitHelperProjectPath%" + if errorlevel 1 ( + exit /b 1 + ) + + :: Build the test helper projects + "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_TestFileHelperProjectPath%" + if errorlevel 1 ( + exit /b 1 + ) + + "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_XunitHelperProjectPath%" + if errorlevel 1 ( + exit /b 1 + ) + + set TESTS_SEMAPHORE=%CoreRT_TestExtRepo_CoreFX%\init-tests.completed + :: If sempahore exists do nothing + if exist "%TESTS_SEMAPHORE%" ( + echo Tests are already initialized. + goto :EOF + ) + + set /p TESTS_REMOTE_URL=< "%~dp0/CoreFXTestListURL.txt" + set TEST_LIST="%~dp0/TopN.CoreFX.issues.json" + + if not exist !CoreRT_TestingUtilitiesOutputDir!\%CoreRT_TestFileHelperName%.dll ( + echo File !CoreRT_TestingUtilitiesOutputDir!\%CoreRT_TestFileHelperName%.dll not found. + exit /b 1 + ) + + "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestingUtilitiesOutputDir!\%CoreRT_TestFileHelperName%.dll --clean --outputDirectory !CoreRT_TestExtRepo_CoreFX! --testListJsonPath "%TEST_LIST%" --testUrl "%TESTS_REMOTE_URL%" + + + exit /b 0 + :RestoreCoreCLRTests - set TESTS_SEMAPHORE=%CoreRT_TestExtRepo%\init-tests.completed + set TESTS_SEMAPHORE=%CoreRT_TestExtRepo_CoreCLR%\init-tests.completed :: If sempahore exists do nothing if exist "%TESTS_SEMAPHORE%" ( @@ -350,15 +397,15 @@ goto :eof goto :EOF ) - if exist "%CoreRT_TestExtRepo%" rmdir /S /Q "%CoreRT_TestExtRepo%" - mkdir "%CoreRT_TestExtRepo%" + if exist "%CoreRT_TestExtRepo_CoreCLR%" rmdir /S /Q "%CoreRT_TestExtRepo_CoreCLR%" + mkdir "%CoreRT_TestExtRepo_CoreCLR%" set /p TESTS_REMOTE_URL=< "%~dp0\CoreCLRTestsURL.txt" - set TESTS_LOCAL_ZIP=%CoreRT_TestExtRepo%\tests.zip + set TESTS_LOCAL_ZIP=%CoreRT_TestExtRepo_CoreCLR%\tests.zip set INIT_TESTS_LOG=%~dp0..\init-tests.log echo Restoring tests (this may take a few minutes).. echo Installing '%TESTS_REMOTE_URL%' to '%TESTS_LOCAL_ZIP%' >> "%INIT_TESTS_LOG%" - powershell -NoProfile -ExecutionPolicy unrestricted -Command "$retryCount = 0; $success = $false; do { try { [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12; (New-Object Net.WebClient).DownloadFile('%TESTS_REMOTE_URL%', '%TESTS_LOCAL_ZIP%'); $success = $true; } catch { if ($retryCount -ge 6) { throw; } else { $retryCount++; Start-Sleep -Seconds (5 * $retryCount); } } } while ($success -eq $false); Add-Type -Assembly 'System.IO.Compression.FileSystem' -ErrorVariable AddTypeErrors; if ($AddTypeErrors.Count -eq 0) { [System.IO.Compression.ZipFile]::ExtractToDirectory('%TESTS_LOCAL_ZIP%', '%CoreRT_TestExtRepo%') } else { (New-Object -com shell.application).namespace('%CoreRT_TestExtRepo%').CopyHere((new-object -com shell.application).namespace('%TESTS_LOCAL_ZIP%').Items(),16) }" >> "%INIT_TESTS_LOG%" + powershell -NoProfile -ExecutionPolicy unrestricted -Command "$retryCount = 0; $success = $false; do { try { [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12; (New-Object Net.WebClient).DownloadFile('%TESTS_REMOTE_URL%', '%TESTS_LOCAL_ZIP%'); $success = $true; } catch { if ($retryCount -ge 6) { throw; } else { $retryCount++; Start-Sleep -Seconds (5 * $retryCount); } } } while ($success -eq $false); Add-Type -Assembly 'System.IO.Compression.FileSystem' -ErrorVariable AddTypeErrors; if ($AddTypeErrors.Count -eq 0) { [System.IO.Compression.ZipFile]::ExtractToDirectory('%TESTS_LOCAL_ZIP%', '%CoreRT_TestExtRepo_CoreCLR%') } else { (New-Object -com shell.application).namespace('%CoreRT_TestExtRepo_CoreCLR%').CopyHere((new-object -com shell.application).namespace('%TESTS_LOCAL_ZIP%').Items(),16) }" >> "%INIT_TESTS_LOG%" if errorlevel 1 ( echo ERROR: Could not download CoreCLR tests correctly. See '%INIT_TESTS_LOG%' for more details. 1>&2 exit /b 1 @@ -368,7 +415,7 @@ goto :eof echo CoreCLR tests restored from %TESTS_REMOTE_URL% > %TESTS_SEMAPHORE% exit /b 0 -:TestExtRepo +:TestExtRepoCoreCLR :: Omit the exclude parameter to CoreCLR's test harness if we're running all tests set CoreCLRExcludeText=exclude if "%CoreRT_CoreCLRTargetsFile%" == "" ( @@ -376,15 +423,15 @@ goto :eof ) echo Running external tests - if "%CoreRT_TestExtRepo%" == "" ( - set CoreRT_TestExtRepo=%CoreRT_TestRoot%\..\tests_downloaded\CoreCLR + if "%CoreRT_TestExtRepo_CoreCLR%" == "" ( + set CoreRT_TestExtRepo_CoreCLR=%CoreRT_TestRoot%\..\tests_downloaded\CoreCLR call :RestoreCoreCLRTests if errorlevel 1 ( exit /b 1 ) ) - if not exist "%CoreRT_TestExtRepo%" ((call :Fail "%CoreRT_TestExtRepo% does not exist") & exit /b 1) + if not exist "%CoreRT_TestExtRepo_CoreCLR%" ((call :Fail "%CoreRT_TestExtRepo_CoreCLR% does not exist") & exit /b 1) if "%CoreRT_MultiFileConfiguration%" == "MultiModule" ( set IlcMultiModule=true @@ -396,7 +443,7 @@ goto :eof echo. set CLRCustomTestLauncher=%CoreRT_TestRoot%\CoreCLR\build-and-run-test.cmd - set XunitTestBinBase=!CoreRT_TestExtRepo! + set XunitTestBinBase=!CoreRT_TestExtRepo_CoreCLR! pushd %CoreRT_TestRoot%\CoreCLR\runtest "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore /p:RepoLocalBuild=true src\TestWrappersConfig\XUnitTooling.depproj @@ -423,3 +470,36 @@ goto :eof set __SavedErrorLevel=%ErrorLevel% popd exit /b %__SavedErrorLevel% + +:TestExtRepoCoreFX + + set CoreRT_TestExtRepo_CoreFX=%CoreRT_TestRoot%\..\tests_downloaded\CoreFX + set CoreRT_TestingUtilitiesOutputDir=%CoreRT_TestExtRepo_CoreFX%\Utilities + + :: Set paths to helpers + set CoreRT_TestFileHelperName=CoreFX.TestUtils.TestFileSetup + set CoreRT_TestFileHelperProjectPath="%CoreRT_TestRoot%\CoreFX\runtest\src\TestUtils\TestFileSetup\%CoreRT_TestFileHelperName%.csproj" + + set CoreRT_XunitHelperName=CoreFX.TestUtils.XUnit + set CoreRT_XunitHelperProjectPath="%CoreRT_TestRoot%\CoreFX\runtest\src\TestUtils\XUnit\%CoreRT_XunitHelperName%.csproj" + + :: TODO Check if each requested test has already been restored + call :RestoreCoreFXTests + if errorlevel 1 ( + exit /b 1 + ) + + set FXCustomTestLauncher=%CoreRT_TestRoot%\CoreFX\build-and-run-test.cmd + set XunitTestBinBase=%CoreRT_TestExtRepo_CoreFX% + set XunitLogDir= %__LogDir%\CoreFX + pushd %CoreRT_TestRoot%\CoreFX\runtest + + :: TODO Add single test/target test support; add exclude tests argument + + echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% + call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% + if errorlevel 1 ( + exit /b 1 + ) + + "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestingUtilitiesOutputDir!\!CoreRT_XunitHelperName!.dll --logDir "%XunitLogDir%" --pattern "*.xml" diff --git a/tests/runtest.sh b/tests/runtest.sh index 51104cf8124..54539993e3f 100755 --- a/tests/runtest.sh +++ b/tests/runtest.sh @@ -6,6 +6,7 @@ usage() echo " -mode : Compilation mode. Specify cpp/ryujit. Default: ryujit" echo " -test : Run a single test by folder name (ie, BasicThreading)" echo " -runtest : Should just compile or run compiled binary? Specify: true/false. Default: true." + echo " -corefx : Download and run the CoreFX repo tests" echo " -coreclr : Download and run the CoreCLR repo tests" echo " -multimodule : Compile the framework as a .so and link tests against it (ryujit only)" echo " -coredumps : [For CI use] Enables core dump generation, and analyzes and possibly stores/uploads" @@ -84,7 +85,7 @@ run_test_dir() return $? } -download_and_unzip_tests_artifacts() +download_and_unzip_coreclr_tests_artifacts() { url=$1 location=$2 @@ -104,6 +105,49 @@ download_and_unzip_tests_artifacts() fi } +download_and_unzip_corefx_tests_artifacts() +{ + url=$1 + test_list=$2 + #semaphore=${CoreRT_TestExtRepo_CoreFX}/init-tests.completed + + local __msbuild_dir=${CoreRT_CliBinDir}/.. + + # Build and restore test helper projects + ${CoreRT_CliBinDir}/dotnet restore "${CoreRT_TestFileHelperProjectPath}" + local __exitcode=$? + if [ ${__exitcode} != 0 ]; then + exit ${__exitcode} + fi + + ${CoreRT_CliBinDir}/dotnet restore "${CoreRT_XunitHelperProjectPath}" + __exitcode=$? + if [ ${__exitcode} != 0 ]; then + exit ${__exitcode} + fi + + # Build the test helper projects + ${CoreRT_CliBinDir}/dotnet build /m /ConsoleLoggerParameters:ForceNoAlign /p:IlcPath=${CoreRT_ToolchainDir} /p:Configuration=${CoreRT_BuildType} /p:Platform=${CoreRT_BuildArch} /p:OSGroup=${CoreRT_BuildOS} /p:RepoLocalBuild=true "/p:FrameworkLibPath=${CoreRT_TestRoot}/../bin/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/lib" "/p:FrameworkObjPath=${CoreRT_TestRoot}/../bin/obj/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/Framework" "/p:OutputPath=${CoreRT_TestingUtilitiesOutputDir}" "${CoreRT_TestFileHelperProjectPath}" + + __exitcode=$? + if [ ${__exitcode} != 0 ]; then + exit ${__exitcode} + fi + + ${CoreRT_CliBinDir}/dotnet build /m /ConsoleLoggerParameters:ForceNoAlign /p:IlcPath=${CoreRT_ToolchainDir} /p:Configuration=${CoreRT_BuildType} /p:Platform=${CoreRT_BuildArch} /p:OSGroup=${CoreRT_BuildOS} /p:RepoLocalBuild=true "/p:FrameworkLibPath=${CoreRT_TestRoot}/../bin/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/lib" "/p:FrameworkObjPath=${CoreRT_TestRoot}/../bin/obj/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/Framework" "/p:OutputPath=${CoreRT_TestingUtilitiesOutputDir}" "${CoreRT_XunitHelperProjectPath}" + __exitcode=$? + if [ ${__exitcode} != 0 ]; then + exit ${__exitcode} + fi + + ${CoreRT_CliBinDir}/dotnet ${CoreRT_TestingUtilitiesOutputDir}/${CoreRT_TestFileHelperName}.dll --clean --outputDirectory ${CoreRT_TestExtRepo_CoreFX} --testListJsonPath ${test_list} --testUrl ${url} + __exitcode=$? + if [ ${__exitcode} != 0 ]; then + exit ${__exitcode} + fi +} + + restore_coreclr_tests() { CoreRT_Test_Download_Semaphore=${CoreRT_TestExtRepo}/init-tests.completed @@ -118,10 +162,10 @@ restore_coreclr_tests() CoreRT_NativeArtifactRepo=${CoreRT_TestExtRepo}/native echo "Restoring tests (this may take a few minutes).." - download_and_unzip_tests_artifacts ${TESTS_REMOTE_URL} ${CoreRT_TestExtRepo} ${CoreRT_Test_Download_Semaphore} + download_and_unzip_coreclr_tests_artifacts ${TESTS_REMOTE_URL} ${CoreRT_TestExtRepo} ${CoreRT_Test_Download_Semaphore} echo "Restoring native test artifacts..." - download_and_unzip_tests_artifacts ${NATIVE_REMOTE_URL} ${CoreRT_NativeArtifactRepo} ${CoreRT_NativeArtifact_Download_Semaphore} + download_and_unzip_coreclr_tests_artifacts ${NATIVE_REMOTE_URL} ${CoreRT_NativeArtifactRepo} ${CoreRT_NativeArtifact_Download_Semaphore} } run_coreclr_tests() @@ -159,6 +203,58 @@ run_coreclr_tests() ./runtest.sh --testRootDir=${CoreRT_TestExtRepo} --coreOverlayDir=${CoreRT_TestRoot}/CoreCLR ${CoreRT_TestSelectionArg} --logdir=$__LogDir --disableEventLogging } +run_corefx_tests() +{ + CoreRT_TestExtRepo_CoreFX=${CoreRT_TestRoot}/../tests_downloaded/CoreFX + CoreRT_TestingUtilitiesOutputDir=${CoreRT_TestExtRepo_CoreFX}/Utilities + + export CoreRT_TestRoot + export CoreRT_EnableCoreDumps + + export CoreRT_TestExtRepo_CoreFX + export CoreRT_TestingUtilitiesOutputDir + export CoreRT_CliBinDir + + # Set paths to helpers + CoreRT_TestFileHelperName=CoreFX.TestUtils.TestFileSetup + CoreRT_TestFileHelperProjectPath="${CoreRT_TestRoot}/CoreFX/runtest/src/TestUtils/TestFileSetup/${CoreRT_TestFileHelperName}.csproj" + + CoreRT_XunitHelperName=CoreFX.TestUtils.XUnit + CoreRT_XunitHelperProjectPath="${CoreRT_TestRoot}/CoreFX/runtest/src/TestUtils/XUnit/${CoreRT_XunitHelperName}.csproj" + + TESTS_REMOTE_URL=$(<${CoreRT_TestRoot}/CoreFXTestListURL.txt) + TEST_LIST_JSON=${CoreRT_TestRoot}/TopN.CoreFX.issues.json + + download_and_unzip_corefx_tests_artifacts ${TESTS_REMOTE_URL} ${TEST_LIST_JSON} + __exitcode=$? + if [ ${__exitcode} != 0 ]; + then + exit ${__exitcode} + fi + + FXCustomTestLauncher=${CoreRT_TestRoot}/CoreFX/corerun + XunitTestBinBase=${CoreRT_TestExtRepo_CoreFX} + XunitLogDir=${__LogDir}/CoreFX + if [ ! -d "${XunitLogDir}" ]; then + mkdir ${XunitLogDir} + fi + + pushd ${CoreRT_TestRoot}/CoreFX/runtest + + # TODO Add single test/target test support; add exclude tests argument + ./runtest.sh --testRootDir=${XunitTestBinBase} --logdir=${XunitLogDir} --testLauncher=${FXCustomTestLauncher} + __exitcode=$? + if [ ${__exitcode} != 0 ]; + then + exit ${__exitcode} + fi + + echo ${CoreRT_CliBinDir}/dotnet ${CoreRT_TestingUtilitiesOutputDir}/${CoreRT_XunitHelperName}.dll --logDir ${XunitLogDir} --pattern "*.xml" + + ${CoreRT_CliBinDir}/dotnet ${CoreRT_TestingUtilitiesOutputDir}/${CoreRT_XunitHelperName}.dll --logDir ${XunitLogDir} --pattern "*.xml" + +} + CoreRT_TestRoot="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CoreRT_CliBinDir=${CoreRT_TestRoot}/../Tools/dotnetcli CoreRT_BuildArch=x64 @@ -238,6 +334,10 @@ while [ "$1" != "" ]; do exit -1 fi ;; + -corefx) + CoreRT_RunCoreFXTests=true; + shift + ;; -multimodule) CoreRT_MultiFileConfiguration=MultiModule; ;; @@ -324,6 +424,11 @@ if [ ${CoreRT_RunCoreCLRTests} ]; then exit $? fi +if [ ${CoreRT_RunCoreFXTests} ]; then + run_corefx_tests + exit $? +fi + __CppTotalTests=0 __CppPassedTests=0 __JitTotalTests=0