From 1c4a957dafd3c48f809f6329f890aad09d9220cb Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Tue, 16 Jan 2018 15:20:10 -0800 Subject: [PATCH 01/16] Generate Test projects - Intermittent checkin --- tests/CoreFX/Test.csproj | 43 ++++++++ tests/CoreFX/build-and-run-test.cmd | 35 +++++++ tests/CoreFX/default.rd.xml | 39 ++++++++ tests/CoreFX/dependencies.props | 95 ++++++++++++++++++ tests/CoreFX/runtest/runtest.cmd | 99 +++++++++++++++++++ .../TestWrappersConfig/XUnitTooling.depproj | 62 ++++++++++++ tests/Top200.CoreFX.issues.targets | 0 tests/runtest.cmd | 56 ++++++++--- 8 files changed, 418 insertions(+), 11 deletions(-) create mode 100644 tests/CoreFX/Test.csproj create mode 100644 tests/CoreFX/build-and-run-test.cmd create mode 100644 tests/CoreFX/default.rd.xml create mode 100644 tests/CoreFX/dependencies.props create mode 100644 tests/CoreFX/runtest/runtest.cmd create mode 100644 tests/CoreFX/runtest/src/TestWrappersConfig/XUnitTooling.depproj create mode 100644 tests/Top200.CoreFX.issues.targets diff --git a/tests/CoreFX/Test.csproj b/tests/CoreFX/Test.csproj new file mode 100644 index 00000000000..3e23fb679da --- /dev/null +++ b/tests/CoreFX/Test.csproj @@ -0,0 +1,43 @@ + + + $(ExecutableName) + .exe + Exe + $(MSBuildProjectDirectory)\ + $(MSBuildProjectDirectory)\ + default.rd.xml + 2.2.0-beta2-build3300 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd new file mode 100644 index 00000000000..b058fb49d5e --- /dev/null +++ b/tests/CoreFX/build-and-run-test.cmd @@ -0,0 +1,35 @@ +:: Test execution wrapper for CoreCLR tests +:: +:: This wrapper is called from each CoreCLR test's .cmd / .sh run script as the custom test launcher. +:: We use this opportunity to invoke the CoreRT compiler and then run the produced native binary. +:: +:: %1 contains test folder +:: %2 contains test exe name +:: +@echo OFF +setlocal ENABLEDELAYEDEXPANSION + +set TestFolder=%1 + +:: +:: We're only interested in referencing the xunit runner - the test dlls will be imported by the test project +:: +set TestExecutable=xunit.console.netcore +set TestFileName=%2 + +echo %TestExecutable% +echo %TestFileName% + +:: Copy artefacts necessary to compile and run the xunit exe +copy /Y %~dp0\Test.csproj %TestFolder% +copy /Y %~dp0\default.rd.xml %TestFolder% + +if "%CoreRT_BuildArch%" == "x64" ( + call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" +) + +echo msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=true" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj +call msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=true" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj + +echo %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll +call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll \ No newline at end of file diff --git a/tests/CoreFX/default.rd.xml b/tests/CoreFX/default.rd.xml new file mode 100644 index 00000000000..b2a34a5737f --- /dev/null +++ b/tests/CoreFX/default.rd.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/CoreFX/dependencies.props b/tests/CoreFX/dependencies.props new file mode 100644 index 00000000000..eb179beec77 --- /dev/null +++ b/tests/CoreFX/dependencies.props @@ -0,0 +1,95 @@ + + + + 9f9f1cf6e6c4a969e8d0158936427b9896300035 + 9f9f1cf6e6c4a969e8d0158936427b9896300035 + 9f9f1cf6e6c4a969e8d0158936427b9896300035 + + + + + beta-24530-02 + beta-24523-00 + + + + + 1.2.0-beta-24529-03 + 2.2.0-beta2-build3300 + + + + + build-info/dotnet/ + master + $(MSBuildThisFileFullPath) + + + + + $(BaseDotNetBuildInfo)corefx/$(DependencyBranch) + $(CoreFxCurrentRef) + + + $(BaseDotNetBuildInfo)coreclr/$(DependencyBranch) + $(CoreClrCurrentRef) + + + $(BaseDotNetBuildInfo)projectk-tfs/$(DependencyBranch) + $(ExternalCurrentRef) + + + + https://raw.githubusercontent.com/dotnet/versions + + + + $(MSBuildThisFileFullPath) + CoreFxExpectedPrerelease + CoreFx + + + $(MSBuildThisFileFullPath) + CoreClrPackageVersion + Microsoft.NETCore.Runtime.CoreCLR + + + $(MSBuildThisFileFullPath) + ExternalExpectedPrerelease + External + + + + + + + + + + + + + $(XunitPackageVersion) + + + + + + + + + + + 1.0.0-alpha-build0040 + + + + 1.0.2-prerelease-00177 + + + + %(Identity) + true + + + diff --git a/tests/CoreFX/runtest/runtest.cmd b/tests/CoreFX/runtest/runtest.cmd new file mode 100644 index 00000000000..237a835654e --- /dev/null +++ b/tests/CoreFX/runtest/runtest.cmd @@ -0,0 +1,99 @@ +@if not defined __echo @echo off +setlocal EnableDelayedExpansion + +:: Set the default arguments +set __BuildArch=x64 +set __BuildType=Debug +set __BuildOS=Windows_NT + +:: Default to highest Visual Studio version available +set __VSVersion=vs2017 +set __VSProductVersion=150 + +:: Define a prefix for most output progress messages that come from this script. That makes +:: it easier to see where these are coming from. Note that there is a trailing space here. +set __MsgPrefix=RUNTEST: + +set __ProjectDir=%~dp0 +:: remove trailing slash +if %__ProjectDir:~-1%==\ set "__ProjectDir=%__ProjectDir:~0,-1%" +set "__ProjectFilesDir=%__ProjectDir%" +set "__RootBinDir=%__ProjectDir%\..\bin" +set "__LogsDir=%__RootBinDir%\Logs" + +:Arg_Loop +if "%1" == "" goto ArgsDone + +if /i "%1" == "/?" goto Usage +if /i "%1" == "-?" goto Usage +if /i "%1" == "/h" goto Usage +if /i "%1" == "-h" goto Usage +if /i "%1" == "/help" goto Usage +if /i "%1" == "-help" goto Usage + +if /i "%1" == "x64" (set __BuildArch=x64&set __MSBuildBuildArch=x64&shift&goto Arg_Loop) +if /i "%1" == "x86" (set __BuildArch=x86&set __MSBuildBuildArch=x86&shift&goto Arg_Loop) + +if /i "%1" == "debug" (set __BuildType=Debug&shift&goto Arg_Loop) +if /i "%1" == "release" (set __BuildType=Release&shift&goto Arg_Loop) + +if /i "%1" == "LogsDir" (set __LogsDir=%2&shift&shift&goto Arg_Loop) + +if /i not "%1" == "msbuildargs" goto SkipMsbuildArgs +:: All the rest of the args will be collected and passed directly to msbuild. +:CollectMsbuildArgs +shift +if "%1"=="" goto ArgsDone +set __msbuildExtraArgs=%__msbuildExtraArgs% %1 +goto CollectMsbuildArgs +:SkipMsbuildArgs + +set CORE_ROOT=%1 +echo %__MsgPrefix%CORE_ROOT is initially set to: "%CORE_ROOT%" +shift +:ArgsDone + +set "__TestWorkingDir=%CoreRT_TestRoot\CoreFX%" + +if not defined XunitTestBinBase set XunitTestBinBase=%__TestWorkingDir% + +if not exist %__LogsDir% md %__LogsDir% + +:: Check presence of VS +if not defined VS%__VSProductVersion%COMNTOOLS goto NoVS + +set __VSToolsRoot=!VS%__VSProductVersion%COMNTOOLS! +if %__VSToolsRoot:~-1%==\ set "__VSToolsRoot=%__VSToolsRoot:~0,-1%" + +set _msbuildexe="%VSINSTALLDIR%\MSBuild\15.0\Bin\MSBuild.exe" + +if not exist !_msbuildexe! (echo Error: Could not find MSBuild.exe. Please see https://github.com/dotnet/corert/blob/master/Documentation/prerequisites-for-building.md for build instructions. && exit /b 1) + +:: Set the environment for the build- VS cmd prompt +echo %__MsgPrefix%Using environment: "%__VSToolsRoot%\VsDevCmd.bat" +call "%__VSToolsRoot%\VsDevCmd.bat" + +if not defined VSINSTALLDIR ( + echo %__MsgPrefix%Error: runtest.cmd should be run from a Visual Studio Command Prompt. Please see https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/developer-guide.md for build instructions. + exit /b 1 +) + +:: Note: We've disabled node reuse because it causes file locking issues. +:: The issue is that we extend the build with our own targets which +:: means that that rebuilding cannot successfully delete the task +:: assembly. +set __msbuildCommonArgs=/nologo /nodeReuse:false %__msbuildExtraArgs% /p:Platform=%__MSBuildBuildArch% + +if not defined __Sequential ( + set __msbuildCommonArgs=%__msbuildCommonArgs% /maxcpucount +) else ( + set __msbuildCommonArgs=%__msbuildCommonArgs% /p:ParallelRun=false +) + +:: Iterate through unzipped CoreFX tests +for /D %%i in ("%XunitTestBinBase%\*" ) do ( + set TestFolderName=%%i + set TestFileName=%%~nxi + echo %FXCustomTestLauncher% !TestFolderName! !TestFileName! + call %FXCustomTestLauncher% !TestFolderName! !TestFileName! +) \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/TestWrappersConfig/XUnitTooling.depproj b/tests/CoreFX/runtest/src/TestWrappersConfig/XUnitTooling.depproj new file mode 100644 index 00000000000..928310a416e --- /dev/null +++ b/tests/CoreFX/runtest/src/TestWrappersConfig/XUnitTooling.depproj @@ -0,0 +1,62 @@ + + + + net46 + 2.2.0-beta2-build3300 + 2.1.0-prerelease-02411-04 + 2.0.1 + + + + + + + + + + + $(XunitPackageVersion) + + + $(XunitPackageVersion) + + + $(XunitPackageVersion) + + + $(XunitPackageVersion) + + + $(XunitAbstractionsVersion) + + + $(XunitPackageVersion) + + + $(XunitPackageVersion) + + + + + + + + + + + + + + + + + diff --git a/tests/Top200.CoreFX.issues.targets b/tests/Top200.CoreFX.issues.targets new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/runtest.cmd b/tests/runtest.cmd index 0c23b18a74b..b9307e10613 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -10,6 +10,7 @@ set CoreRT_BuildOS=Windows_NT set CoreRT_TestRun=true set CoreRT_TestCompileMode= set CoreRT_RunCoreCLRTests= +set CoreRT_RunCoreFXTests= set CoreRT_CoreCLRTargetsFile= set CoreRT_TestLogFileName=testresults.xml set CoreRT_TestName=* @@ -48,6 +49,7 @@ if /i "%1" == "/coreclr" ( :ExtRepoTestsOk goto ArgLoop ) +if /i "%1" == "/corefx" (set CoreRT_RunCoreFXTests=true&shift&goto ArgLoop) if /i "%1" == "/coreclrsingletest" (set CoreRT_RunCoreCLRTests=true&set CoreRT_CoreCLRTest=%2&shift&shift&goto ArgLoop) if /i "%1" == "/mode" (set CoreRT_TestCompileMode=%2&shift&shift&goto ArgLoop) if /i "%1" == "/test" (set CoreRT_TestName=%2&shift&shift&goto ArgLoop) @@ -67,6 +69,7 @@ echo /mode : Optionally restrict to a single code generator. Specify echo /test : Run a single test by folder name (ie, BasicThreading) echo /runtest : Should just compile or run compiled binary? Specify: true/false. Default: true. echo /coreclr : Download and run the CoreCLR repo tests +echo /corefx : Download and run the CoreFX repo tests echo /coreclrsingletest ^ echo : Run a single CoreCLR repo test echo /multimodule : Compile the framework as a .lib and link tests against it (only supports ryujit) @@ -121,7 +124,8 @@ call "%_VSCOMNTOOLS%\VsDevCmd.bat" :RunVCVars call "!VS150COMNTOOLS!\..\..\VC\Auxiliary\Build\vcvarsall.bat" %CoreRT_HostArch% -if "%CoreRT_RunCoreCLRTests%"=="true" goto :TestExtRepo +if "%CoreRT_RunCoreCLRTests%"=="true" goto :TestExtRepoCoreCLR +if "%CoreRT_RunCoreFXTests%"=="true" goto :TestExtRepoCoreFX if /i "%__BuildType%"=="Debug" ( set __LinkLibs=msvcrtd.lib @@ -333,9 +337,14 @@ goto :eof powershell -Command Write-Host %1 -foreground "red" exit /b -1 +:RestoreCoreFXTests + +echo todo +exit /b 0 + :RestoreCoreCLRTests - set TESTS_SEMAPHORE=%CoreRT_TestExtRepo%\init-tests.completed + set TESTS_SEMAPHORE=%CoreRT_TestExtRepo_CoreCLR%\init-tests.completed :: If sempahore exists do nothing if exist "%TESTS_SEMAPHORE%" ( @@ -343,15 +352,15 @@ goto :eof goto :EOF ) - if exist "%CoreRT_TestExtRepo%" rmdir /S /Q "%CoreRT_TestExtRepo%" - mkdir "%CoreRT_TestExtRepo%" + if exist "%CoreRT_TestExtRepo_CoreCLR%" rmdir /S /Q "%CoreRT_TestExtRepo_CoreCLR%" + mkdir "%CoreRT_TestExtRepo_CoreCLR%" set /p TESTS_REMOTE_URL=< "%~dp0\CoreCLRTestsURL.txt" - set TESTS_LOCAL_ZIP=%CoreRT_TestExtRepo%\tests.zip + set TESTS_LOCAL_ZIP=%CoreRT_TestExtRepo_CoreCLR%\tests.zip set INIT_TESTS_LOG=%~dp0..\init-tests.log echo Restoring tests (this may take a few minutes).. echo Installing '%TESTS_REMOTE_URL%' to '%TESTS_LOCAL_ZIP%' >> "%INIT_TESTS_LOG%" - powershell -NoProfile -ExecutionPolicy unrestricted -Command "$retryCount = 0; $success = $false; do { try { [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12; (New-Object Net.WebClient).DownloadFile('%TESTS_REMOTE_URL%', '%TESTS_LOCAL_ZIP%'); $success = $true; } catch { if ($retryCount -ge 6) { throw; } else { $retryCount++; Start-Sleep -Seconds (5 * $retryCount); } } } while ($success -eq $false); Add-Type -Assembly 'System.IO.Compression.FileSystem' -ErrorVariable AddTypeErrors; if ($AddTypeErrors.Count -eq 0) { [System.IO.Compression.ZipFile]::ExtractToDirectory('%TESTS_LOCAL_ZIP%', '%CoreRT_TestExtRepo%') } else { (New-Object -com shell.application).namespace('%CoreRT_TestExtRepo%').CopyHere((new-object -com shell.application).namespace('%TESTS_LOCAL_ZIP%').Items(),16) }" >> "%INIT_TESTS_LOG%" + powershell -NoProfile -ExecutionPolicy unrestricted -Command "$retryCount = 0; $success = $false; do { try { [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12; (New-Object Net.WebClient).DownloadFile('%TESTS_REMOTE_URL%', '%TESTS_LOCAL_ZIP%'); $success = $true; } catch { if ($retryCount -ge 6) { throw; } else { $retryCount++; Start-Sleep -Seconds (5 * $retryCount); } } } while ($success -eq $false); Add-Type -Assembly 'System.IO.Compression.FileSystem' -ErrorVariable AddTypeErrors; if ($AddTypeErrors.Count -eq 0) { [System.IO.Compression.ZipFile]::ExtractToDirectory('%TESTS_LOCAL_ZIP%', '%CoreRT_TestExtRepo_CoreCLR%') } else { (New-Object -com shell.application).namespace('%CoreRT_TestExtRepo_CoreCLR%').CopyHere((new-object -com shell.application).namespace('%TESTS_LOCAL_ZIP%').Items(),16) }" >> "%INIT_TESTS_LOG%" if errorlevel 1 ( echo ERROR: Could not download CoreCLR tests correctly. See '%INIT_TESTS_LOG%' for more details. 1>&2 exit /b 1 @@ -361,7 +370,7 @@ goto :eof echo CoreCLR tests restored from %TESTS_REMOTE_URL% > %TESTS_SEMAPHORE% exit /b 0 -:TestExtRepo +:TestExtRepoCoreCLR :: Omit the exclude parameter to CoreCLR's test harness if we're running all tests set CoreCLRExcludeText=exclude if "%CoreRT_CoreCLRTargetsFile%" == "" ( @@ -369,15 +378,15 @@ goto :eof ) echo Running external tests - if "%CoreRT_TestExtRepo%" == "" ( - set CoreRT_TestExtRepo=%CoreRT_TestRoot%\..\tests_downloaded\CoreCLR + if "%CoreRT_TestExtRepo_CoreCLR%" == "" ( + set CoreRT_TestExtRepo_CoreCLR=%CoreRT_TestRoot%\..\tests_downloaded\CoreCLR call :RestoreCoreCLRTests if errorlevel 1 ( exit /b 1 ) ) - if not exist "%CoreRT_TestExtRepo%" ((call :Fail "%CoreRT_TestExtRepo% does not exist") & exit /b 1) + if not exist "%CoreRT_TestExtRepo_CoreCLR%" ((call :Fail "%CoreRT_TestExtRepo_CoreCLR% does not exist") & exit /b 1) if "%CoreRT_MultiFileConfiguration%" == "MultiModule" ( set IlcMultiModule=true @@ -389,7 +398,7 @@ goto :eof echo. set CLRCustomTestLauncher=%CoreRT_TestRoot%\CoreCLR\build-and-run-test.cmd - set XunitTestBinBase=!CoreRT_TestExtRepo! + set XunitTestBinBase=!CoreRT_TestExtRepo_CoreCLR! pushd %CoreRT_TestRoot%\CoreCLR\runtest "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore /p:RepoLocalBuild=true src\TestWrappersConfig\XUnitTooling.depproj @@ -416,3 +425,28 @@ goto :eof set __SavedErrorLevel=%ErrorLevel% popd exit /b %__SavedErrorLevel% + +:TestExtRepoCoreFX + + if "%CoreRT_TestExtRepo_CoreFX%" == "" ( + set CoreRT_TestExtRepo_CoreFX=%CoreRT_TestRoot%\..\tests_downloaded\CoreFX + call :RestoreCoreFXTests + if errorlevel 1 ( + exit /b 1 + ) + ) + + set FXCustomTestLauncher=%CoreRT_TestRoot%\CoreFX\build-and-run-test.cmd + set XunitTestBinBase=!CoreRT_TestExtRepo_CoreFX! + pushd %CoreRT_TestRoot%\CoreFX\runtest + + "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore /p:RepoLocalBuild=true src\TestWrappersConfig\XUnitTooling.depproj + if errorlevel 1 ( + exit /b 1 + ) + + rem TODO Add single test/target test support; add exclude tests + + echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% + call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% + \ No newline at end of file From c5c39011899f9fddb40d34aa71b83721db725055 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Tue, 16 Jan 2018 19:20:46 -0800 Subject: [PATCH 02/16] Intermittent check-in running exe, tests need restoring --- tests/CoreFX/Test.csproj | 25 +++----- tests/CoreFX/build-and-run-test.cmd | 4 ++ tests/CoreFX/default.rd.xml | 3 +- tests/CoreFX/dependencies.props | 95 ----------------------------- tests/CoreFX/runtest/dir.props | 54 ++++++++++++++++ tests/CoreFXTestPayloadURL.txt | 1 + tests/runtest.cmd | 12 +++- 7 files changed, 78 insertions(+), 116 deletions(-) delete mode 100644 tests/CoreFX/dependencies.props create mode 100644 tests/CoreFX/runtest/dir.props create mode 100644 tests/CoreFXTestPayloadURL.txt diff --git a/tests/CoreFX/Test.csproj b/tests/CoreFX/Test.csproj index 3e23fb679da..2a55a6bc5cb 100644 --- a/tests/CoreFX/Test.csproj +++ b/tests/CoreFX/Test.csproj @@ -8,30 +8,21 @@ $(MSBuildProjectDirectory)\ default.rd.xml 2.2.0-beta2-build3300 + $(FrameworkLibPath)\..\tools\ - - - - - - - - - - + - - + + - - - - + + + + diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index b058fb49d5e..95ffde319ed 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -24,6 +24,10 @@ echo %TestFileName% copy /Y %~dp0\Test.csproj %TestFolder% copy /Y %~dp0\default.rd.xml %TestFolder% +:: Workaround until we have a better reflection engine +:: Add name of currently executing test to rd.xml +powershell -Command "(Get-Content %TestFolder%\default.rd.xml).replace('*Application*', '%TestFileName%') | Set-Content %TestFolder%\default.rd.xml" + if "%CoreRT_BuildArch%" == "x64" ( call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" ) diff --git a/tests/CoreFX/default.rd.xml b/tests/CoreFX/default.rd.xml index b2a34a5737f..f5d7a80a8c5 100644 --- a/tests/CoreFX/default.rd.xml +++ b/tests/CoreFX/default.rd.xml @@ -1,6 +1,7 @@ + @@ -29,7 +30,7 @@ - + diff --git a/tests/CoreFX/dependencies.props b/tests/CoreFX/dependencies.props deleted file mode 100644 index eb179beec77..00000000000 --- a/tests/CoreFX/dependencies.props +++ /dev/null @@ -1,95 +0,0 @@ - - - - 9f9f1cf6e6c4a969e8d0158936427b9896300035 - 9f9f1cf6e6c4a969e8d0158936427b9896300035 - 9f9f1cf6e6c4a969e8d0158936427b9896300035 - - - - - beta-24530-02 - beta-24523-00 - - - - - 1.2.0-beta-24529-03 - 2.2.0-beta2-build3300 - - - - - build-info/dotnet/ - master - $(MSBuildThisFileFullPath) - - - - - $(BaseDotNetBuildInfo)corefx/$(DependencyBranch) - $(CoreFxCurrentRef) - - - $(BaseDotNetBuildInfo)coreclr/$(DependencyBranch) - $(CoreClrCurrentRef) - - - $(BaseDotNetBuildInfo)projectk-tfs/$(DependencyBranch) - $(ExternalCurrentRef) - - - - https://raw.githubusercontent.com/dotnet/versions - - - - $(MSBuildThisFileFullPath) - CoreFxExpectedPrerelease - CoreFx - - - $(MSBuildThisFileFullPath) - CoreClrPackageVersion - Microsoft.NETCore.Runtime.CoreCLR - - - $(MSBuildThisFileFullPath) - ExternalExpectedPrerelease - External - - - - - - - - - - - - - $(XunitPackageVersion) - - - - - - - - - - - 1.0.0-alpha-build0040 - - - - 1.0.2-prerelease-00177 - - - - %(Identity) - true - - - diff --git a/tests/CoreFX/runtest/dir.props b/tests/CoreFX/runtest/dir.props new file mode 100644 index 00000000000..63c226ff273 --- /dev/null +++ b/tests/CoreFX/runtest/dir.props @@ -0,0 +1,54 @@ + + + + + $(OS) + + + + + 1.0.0-rc3-20150510-01 + Microsoft.Net.ToolsetCompilers + + + + + false + $(MSBuildThisFileDirectory)\ + $(ProjectDir)src\ + $(ProjectDir)..\..\..\packages\ + $(ProjectDir)..\Tools\ + $(ToolsDir)dotnetcli/ + true + + + + + + + + + + + + + + + false + + + + $(DotnetCliPath)dotnet.exe + $(DotnetCliPath)dotnet + + @(DnuSourceList -> '--source %(Identity)', ' ') + @(DnuRestoreDir -> '%(Identity)', ' ') + + $(DnuRestoreCommand) "$(DotnetToolCommand)" + $(DnuRestoreCommand) restore + $(DnuRestoreCommand) --packages "$(PackagesDir.TrimEnd('/\'.ToCharArray()))" $(DnuRestoreSource) + + + diff --git a/tests/CoreFXTestPayloadURL.txt b/tests/CoreFXTestPayloadURL.txt new file mode 100644 index 00000000000..f7e95fcf3e0 --- /dev/null +++ b/tests/CoreFXTestPayloadURL.txt @@ -0,0 +1 @@ +https://dotnetbuilddrops.blob.core.windows.net/build-e0a70e19967c4fbfa9e6a9e7fed567cc/TestList.json?sv=2015-04-05&sr=c&sig=rwqOIDM585nrCg4ydSleaYi%2FN1%2BcFqEivySw06MiZRs%3D&st=2018-01-12T01%3A34%3A06.4065828Z&se=2018-02-11T01%3A34%3A06.4065828Z&sp=r \ No newline at end of file diff --git a/tests/runtest.cmd b/tests/runtest.cmd index b9307e10613..ab318f3913d 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -338,9 +338,15 @@ goto :eof exit /b -1 :RestoreCoreFXTests - -echo todo -exit /b 0 + + set TESTS_SEMAPHORE=%CoreRT_TestExtRepo_CoreFX%\init-tests.completed + :: If sempahore exists do nothing + if exist "%TESTS_SEMAPHORE%" ( + echo Tests are already initialized. + goto :EOF + ) + + exit /b 0 :RestoreCoreCLRTests From 6f9bfab20f10f3527d77b1381a6f2b35e46c8a1d Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Thu, 18 Jan 2018 18:11:51 -0800 Subject: [PATCH 03/16] Add Test utitilites project --- tests/CoreFX/runtest.proj | 0 .../CoreFXTestUtils/ArchiveTestUtils.cs | 159 ++++++++++++++++++ .../CoreFXTestUtils/CoreFXTestUtils.csproj | 12 ++ tests/runtest.cmd | 38 +++-- 4 files changed, 198 insertions(+), 11 deletions(-) create mode 100644 tests/CoreFX/runtest.proj create mode 100644 tests/CoreFX/runtest/src/Common/CoreFXTestUtils/ArchiveTestUtils.cs create mode 100644 tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.csproj diff --git a/tests/CoreFX/runtest.proj b/tests/CoreFX/runtest.proj new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/ArchiveTestUtils.cs b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/ArchiveTestUtils.cs new file mode 100644 index 00000000000..dfe6dd69cfc --- /dev/null +++ b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/ArchiveTestUtils.cs @@ -0,0 +1,159 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Net.Http; +using System.Threading.Tasks; +using Newtonsoft.Json; + +namespace CoreFXTestUtils +{ + public static class ArchiveTestUtils + { + static string testUrl = "https://dotnetbuilddrops.blob.core.windows.net/build-e0a70e19967c4fbfa9e6a9e7fed567cc/TestList.json?sv=2015-04-05&sr=c&sig=rwqOIDM585nrCg4ydSleaYi%2FN1%2BcFqEivySw06MiZRs%3D&st=2018-01-12T01%3A34%3A06.4065828Z&se=2018-02-11T01%3A34%3A06.4065828Z&sp=r"; + static Dictionary testNames = new Dictionary(){ { "Common.Tests", "" }, { "System.Collections.Tests", "" }, { "asdasdasd", "" } }; + private static HttpClient httpClient; + + static string baseOutputDir = @"C:\Users\anandono\source\repos\CoreFXTestsDownloaded\"; + + public static void Main(string[] args) + { + if(args.Length < 1) + { + throw new ArgumentException("Please supply an output directory"); + } + + if (!Directory.Exists(baseOutputDir)) + { + Directory.CreateDirectory(baseOutputDir); + } + + // parse args + SetupTests(testNames, testUrl, baseOutputDir); + } + + public static void SetupTests(Dictionary testNames, String jsonUrl, string destinationDirectory) + { + string tempDirPath = Path.Combine(destinationDirectory, "temp"); + if (!Directory.Exists(tempDirPath)) + { + Directory.CreateDirectory(tempDirPath); + } + + Dictionary testPayloads = GetTestUrls(testNames, testUrl).Result; + GetTestArchives(testPayloads, tempDirPath).Wait(); + ExpandArchivesInDirectory(tempDirPath, destinationDirectory); + + Directory.Delete(tempDirPath); + } + + public static async Task> GetTestUrls(Dictionary testNames, string jsonUrl) + { + if(httpClient is null) + { + httpClient = new HttpClient(); + } + + // Set up the json stream reader + using (var responseStream = httpClient.GetStreamAsync(jsonUrl).Result) + using (var streamReader = new StreamReader(responseStream)) + using (var jsonReader = new JsonTextReader(streamReader)) + { + // Manual parsing - we only need to key-value pairs from each object and this avoids deserializing all of the work items into objects + string markedTestName = String.Empty; + string currentPropertyName = String.Empty; + + while (jsonReader.Read()) + { + if(jsonReader.Value != null) + { + switch (jsonReader.TokenType) + { + case JsonToken.PropertyName: + currentPropertyName = jsonReader.Value.ToString(); + break; + case JsonToken.String: + if (currentPropertyName.Equals("WorkItemId")) + { + string currentTestName = jsonReader.Value.ToString(); + // Test if we've added the key + //// TODO add per test fetching + if (testNames.ContainsKey(currentTestName)) + { + markedTestName = currentTestName; + } + } + else if(currentPropertyName.Equals("PayloadUri") && markedTestName != String.Empty) + { + testNames[markedTestName] = jsonReader.Value.ToString(); + markedTestName = String.Empty; + } + break; + } + } + } + + } + return testNames; + } + + public static async Task GetTestArchives(Dictionary testPayloads, string downloadDir) + { + if (httpClient is null) + { + httpClient = new HttpClient(); + } + + foreach(string testName in testPayloads.Keys) + { + string payloadUri = testPayloads[testName]; + + if (!Uri.IsWellFormedUriString(payloadUri, UriKind.Absolute)) + continue; + + using (var response = await httpClient.GetStreamAsync(payloadUri)) + { + if (response.CanRead) + { + // Create the test setup directory if it doesn't exist + if (!Directory.Exists(downloadDir)) + { + Directory.CreateDirectory(downloadDir); + } + string archivePath = Path.Combine(downloadDir, testName + ".zip"); + + // Copy to a temp folder + using (FileStream file = new FileStream(archivePath, FileMode.Create)) + { + await response.CopyToAsync(file); + } + + } + } + } + } + public static void ExpandArchivesInDirectory(string archiveDirectory) + { + ExpandArchivesInDirectory(archiveDirectory, archiveDirectory); + } + + public static void ExpandArchivesInDirectory(string archiveDirectory, string destinationDirectory, bool cleanup = true) + { + string[] archives = Directory.GetFiles(archiveDirectory, "*.zip", SearchOption.TopDirectoryOnly); + + foreach (string archivePath in archives) + { + string destinationDirName = Path.GetFileNameWithoutExtension(archivePath); + ZipFile.ExtractToDirectory(archivePath, destinationDirectory + destinationDirName); + + // Delete archives + if (cleanup) + { + File.Delete(archivePath); + } + } + } + + + } +} diff --git a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.csproj b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.csproj new file mode 100644 index 00000000000..481163d46bb --- /dev/null +++ b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.csproj @@ -0,0 +1,12 @@ + + + netcoreapp2.0 + Exe + + + + + + + + diff --git a/tests/runtest.cmd b/tests/runtest.cmd index ab318f3913d..3f9140d9616 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -339,14 +339,27 @@ goto :eof :RestoreCoreFXTests - set TESTS_SEMAPHORE=%CoreRT_TestExtRepo_CoreFX%\init-tests.completed - :: If sempahore exists do nothing - if exist "%TESTS_SEMAPHORE%" ( - echo Tests are already initialized. - goto :EOF - ) + :: Explicitly restore the test helper project + "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "!CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\CoreFXTestUtils.csproj" + if errorlevel 1 ( + exit /b 1 + ) + + :: Build the test helper project + "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:RepoLocalBuild=true "!CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\CoreFXTestUtils.csproj" + if errorlevel 1 ( + exit /b 1 + ) + + set TESTS_SEMAPHORE=%CoreRT_TestExtRepo_CoreFX%\init-tests.completed + :: If sempahore exists do nothing + if exist "%TESTS_SEMAPHORE%" ( + echo Tests are already initialized. + goto :EOF + ) + "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\bin\!CoreRT_BuildArch!\!CoreRT_BuildType!\netcoreapp2.0\CoreFXTestUtils.dll - exit /b 0 + exit /b 0 :RestoreCoreCLRTests @@ -412,6 +425,7 @@ goto :eof exit /b 1 ) + if not "%CoreRT_CoreCLRTest%" == "" ( if not exist "%CoreRT_CoreCLRTest%" ( echo Target test file not found: %CoreRT_CoreCLRTest% @@ -434,13 +448,15 @@ goto :eof :TestExtRepoCoreFX - if "%CoreRT_TestExtRepo_CoreFX%" == "" ( + + ::if "%CoreRT_TestExtRepo_CoreFX%" == "" ( set CoreRT_TestExtRepo_CoreFX=%CoreRT_TestRoot%\..\tests_downloaded\CoreFX + call :RestoreCoreFXTests if errorlevel 1 ( exit /b 1 ) - ) + ::) set FXCustomTestLauncher=%CoreRT_TestRoot%\CoreFX\build-and-run-test.cmd set XunitTestBinBase=!CoreRT_TestExtRepo_CoreFX! @@ -453,6 +469,6 @@ goto :eof rem TODO Add single test/target test support; add exclude tests - echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% - call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% + REM echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% + REM call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% \ No newline at end of file From d81f654f9595ee8e4e2d15202b8d90d718660404 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Fri, 19 Jan 2018 14:06:38 -0800 Subject: [PATCH 04/16] End-to-End test build Windows --- ...> CoreFXTestUtils.TestArchiveUtils.csproj} | 3 +- ...rchiveTestUtils.cs => TestArchiveUtils.cs} | 120 ++++++++++++++---- ...stPayloadURL.txt => CoreFXTestListURL.txt} | 0 tests/Top200.CoreFX.issues.json | 4 + tests/Top200.CoreFX.issues.targets | 0 tests/runtest.cmd | 13 +- 6 files changed, 108 insertions(+), 32 deletions(-) rename tests/CoreFX/runtest/src/Common/CoreFXTestUtils/{CoreFXTestUtils.csproj => CoreFXTestUtils.TestArchiveUtils.csproj} (96%) rename tests/CoreFX/runtest/src/Common/CoreFXTestUtils/{ArchiveTestUtils.cs => TestArchiveUtils.cs} (54%) rename tests/{CoreFXTestPayloadURL.txt => CoreFXTestListURL.txt} (100%) create mode 100644 tests/Top200.CoreFX.issues.json delete mode 100644 tests/Top200.CoreFX.issues.targets diff --git a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.csproj b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.TestArchiveUtils.csproj similarity index 96% rename from tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.csproj rename to tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.TestArchiveUtils.csproj index 481163d46bb..0d4d803c1af 100644 --- a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.csproj +++ b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.TestArchiveUtils.csproj @@ -6,7 +6,6 @@ - - + \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/ArchiveTestUtils.cs b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs similarity index 54% rename from tests/CoreFX/runtest/src/Common/CoreFXTestUtils/ArchiveTestUtils.cs rename to tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs index dfe6dd69cfc..195702568a7 100644 --- a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/ArchiveTestUtils.cs +++ b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs @@ -1,47 +1,108 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Net.Http; using System.Threading.Tasks; using Newtonsoft.Json; -namespace CoreFXTestUtils +namespace CoreFXTestUtils.TestArchiveUtils { - public static class ArchiveTestUtils + public static class TestArchiveUtils { - static string testUrl = "https://dotnetbuilddrops.blob.core.windows.net/build-e0a70e19967c4fbfa9e6a9e7fed567cc/TestList.json?sv=2015-04-05&sr=c&sig=rwqOIDM585nrCg4ydSleaYi%2FN1%2BcFqEivySw06MiZRs%3D&st=2018-01-12T01%3A34%3A06.4065828Z&se=2018-02-11T01%3A34%3A06.4065828Z&sp=r"; static Dictionary testNames = new Dictionary(){ { "Common.Tests", "" }, { "System.Collections.Tests", "" }, { "asdasdasd", "" } }; private static HttpClient httpClient; - - static string baseOutputDir = @"C:\Users\anandono\source\repos\CoreFXTestsDownloaded\"; + private static bool cleanTestBuild = false; public static void Main(string[] args) { - if(args.Length < 1) + string outputDir = string.Empty; + string testUrl = string.Empty; + string testListPath = string.Empty; + + Console.WriteLine("Passed " + args.Length + " arguments; which are"); + + foreach (string arg in args) { - throw new ArgumentException("Please supply an output directory"); + Console.WriteLine(arg); + } + + for (int i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--outputDirectory": + case "--out": + case "--outputDir": + if(i + 1 < args.Length && !args[i+1].Substring(0,2).Equals("--")) + { + outputDir = args[i + 1]; + i++; + } + break; + case "--testUrl": + if (i + 1 < args.Length && !args[i + 1].Substring(0, 2).Equals("--")) + { + testUrl = args[i + 1]; + i++; + } + break; + case "--testListJsonPath": + if (i + 1 < args.Length && !args[i + 1].Substring(0, 2).Equals("--")) + { + testListPath = args[i + 1]; + i++; + } + break; + case "--clean": + cleanTestBuild = true; + break; + + } } - if (!Directory.Exists(baseOutputDir)) + if (!Directory.Exists(outputDir)) { - Directory.CreateDirectory(baseOutputDir); + Directory.CreateDirectory(outputDir); } - // parse args - SetupTests(testNames, testUrl, baseOutputDir); + SetupTests(testNames, testUrl, outputDir).Wait(); + } + + public static Dictionary ReadTestNames(string testFilePath) + { + Debug.Assert(File.Exists(testFilePath)); + + Dictionary testNames = new Dictionary(); + // We assume we're being a passed a list of test assembly names, so ignore anything that's not a string + using (var sr = new StreamReader(testFilePath)) + using (var jsonReader = new JsonTextReader(sr)) + { + while (jsonReader.Read()) + { + if(jsonReader.TokenType == JsonToken.String) + { + testNames.Add(jsonReader.Value.ToString(), string.Empty); + } + } + } + + return testNames; } - public static void SetupTests(Dictionary testNames, String jsonUrl, string destinationDirectory) + public static async Task SetupTests(Dictionary testNames, string jsonUrl, string destinationDirectory) { + Debug.Assert(Directory.Exists(destinationDirectory)); + string tempDirPath = Path.Combine(destinationDirectory, "temp"); if (!Directory.Exists(tempDirPath)) { Directory.CreateDirectory(tempDirPath); } - Dictionary testPayloads = GetTestUrls(testNames, testUrl).Result; - GetTestArchives(testPayloads, tempDirPath).Wait(); + Dictionary testPayloads = await GetTestUrls(testNames, jsonUrl); + await GetTestArchives(testPayloads, tempDirPath); ExpandArchivesInDirectory(tempDirPath, destinationDirectory); Directory.Delete(tempDirPath); @@ -55,13 +116,13 @@ public static async Task> GetTestUrls(Dictionary> GetTestUrls(Dictionary testPayloads { Directory.CreateDirectory(downloadDir); } + + // CoreFX test archives are output as .zip regardless of platform string archivePath = Path.Combine(downloadDir, testName + ".zip"); // Copy to a temp folder @@ -132,19 +195,26 @@ public static async Task GetTestArchives(Dictionary testPayloads } } } - public static void ExpandArchivesInDirectory(string archiveDirectory) - { - ExpandArchivesInDirectory(archiveDirectory, archiveDirectory); - } public static void ExpandArchivesInDirectory(string archiveDirectory, string destinationDirectory, bool cleanup = true) { + Debug.Assert(Directory.Exists(archiveDirectory)); + Debug.Assert(Directory.Exists(destinationDirectory)); + string[] archives = Directory.GetFiles(archiveDirectory, "*.zip", SearchOption.TopDirectoryOnly); foreach (string archivePath in archives) { - string destinationDirName = Path.GetFileNameWithoutExtension(archivePath); - ZipFile.ExtractToDirectory(archivePath, destinationDirectory + destinationDirName); + string destinationDirName = Path.Combine(destinationDirectory, Path.GetFileNameWithoutExtension(archivePath)); + + // If doing clean test build - delete existing artefacts + if (Directory.Exists(destinationDirName) && cleanTestBuild) + { + Directory.Delete(destinationDirName, true); + } + + ZipFile.ExtractToDirectory(archivePath, destinationDirName); + // Delete archives if (cleanup) diff --git a/tests/CoreFXTestPayloadURL.txt b/tests/CoreFXTestListURL.txt similarity index 100% rename from tests/CoreFXTestPayloadURL.txt rename to tests/CoreFXTestListURL.txt diff --git a/tests/Top200.CoreFX.issues.json b/tests/Top200.CoreFX.issues.json new file mode 100644 index 00000000000..5200c64af57 --- /dev/null +++ b/tests/Top200.CoreFX.issues.json @@ -0,0 +1,4 @@ +[ + "System.Collections.Tests", + "System.Collections.Immutable.Tests" +] \ No newline at end of file diff --git a/tests/Top200.CoreFX.issues.targets b/tests/Top200.CoreFX.issues.targets deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/runtest.cmd b/tests/runtest.cmd index 3f9140d9616..f253b20f34e 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -340,13 +340,13 @@ goto :eof :RestoreCoreFXTests :: Explicitly restore the test helper project - "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "!CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\CoreFXTestUtils.csproj" + "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "!CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\CoreFXTestUtils.TestArchiveUtils.csproj" if errorlevel 1 ( exit /b 1 ) :: Build the test helper project - "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:RepoLocalBuild=true "!CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\CoreFXTestUtils.csproj" + "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:RepoLocalBuild=true "!CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\CoreFXTestUtils.TestArchiveUtils.csproj" if errorlevel 1 ( exit /b 1 ) @@ -357,7 +357,10 @@ goto :eof echo Tests are already initialized. goto :EOF ) - "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\bin\!CoreRT_BuildArch!\!CoreRT_BuildType!\netcoreapp2.0\CoreFXTestUtils.dll + + set /p TESTS_REMOTE_URL=< "%~dp0/CoreFXTestListURL.txt" + set TEST_LIST="%~dp0/Top200.CoreFX.issues.json" + "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\bin\!CoreRT_BuildArch!\!CoreRT_BuildType!\netcoreapp2.0\CoreFXTestUtils.TestArchiveUtils.dll --clean --outputDirectory !CoreRT_TestExtRepo_CoreFX! --testListJsonPath "%TEST_LIST%" --testUrl "%TESTS_REMOTE_URL%" exit /b 0 @@ -469,6 +472,6 @@ goto :eof rem TODO Add single test/target test support; add exclude tests - REM echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% - REM call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% + echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% + call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% \ No newline at end of file From 88a8d07d505a5ac6852c8360a34d33172f7881db Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Fri, 19 Jan 2018 15:20:09 -0800 Subject: [PATCH 05/16] Change structure --- tests/CoreFX/build-and-run-test.cmd | 6 ++---- tests/CoreFX/{ => runtest/CoreFXTestHarness}/Test.csproj | 1 - tests/CoreFX/{ => runtest/CoreFXTestHarness}/default.rd.xml | 3 +-- 3 files changed, 3 insertions(+), 7 deletions(-) rename tests/CoreFX/{ => runtest/CoreFXTestHarness}/Test.csproj (99%) rename tests/CoreFX/{ => runtest/CoreFXTestHarness}/default.rd.xml (93%) diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index 95ffde319ed..b3b9e78d31c 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -17,12 +17,10 @@ set TestFolder=%1 set TestExecutable=xunit.console.netcore set TestFileName=%2 -echo %TestExecutable% -echo %TestFileName% :: Copy artefacts necessary to compile and run the xunit exe -copy /Y %~dp0\Test.csproj %TestFolder% -copy /Y %~dp0\default.rd.xml %TestFolder% +copy /Y "%~dp0\runtest\CoreFXTestHarness\*" "%TestFolder%" + :: Workaround until we have a better reflection engine :: Add name of currently executing test to rd.xml diff --git a/tests/CoreFX/Test.csproj b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj similarity index 99% rename from tests/CoreFX/Test.csproj rename to tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj index 2a55a6bc5cb..607f7870275 100644 --- a/tests/CoreFX/Test.csproj +++ b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj @@ -16,7 +16,6 @@ - diff --git a/tests/CoreFX/default.rd.xml b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml similarity index 93% rename from tests/CoreFX/default.rd.xml rename to tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml index f5d7a80a8c5..eb9c9beaf2c 100644 --- a/tests/CoreFX/default.rd.xml +++ b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml @@ -3,7 +3,6 @@ - @@ -33,7 +32,7 @@ - + From 4c5c1e2720efa0fd1761565ff78f02e646a3da05 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Fri, 19 Jan 2018 18:51:35 -0800 Subject: [PATCH 06/16] Fix RDXML and add reflectable types --- tests/CoreFX/build-and-run-test.cmd | 1 - .../runtest/CoreFXTestHarness/Test.csproj | 25 ++++++++++++++++--- .../runtest/CoreFXTestHarness/default.rd.xml | 13 ++++++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index b3b9e78d31c..6a6d0805489 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -29,7 +29,6 @@ powershell -Command "(Get-Content %TestFolder%\default.rd.xml).replace('*Applica if "%CoreRT_BuildArch%" == "x64" ( call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" ) - echo msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=true" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj call msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=true" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj index 607f7870275..37d551df945 100644 --- a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj +++ b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj @@ -6,18 +6,37 @@ Exe $(MSBuildProjectDirectory)\ $(MSBuildProjectDirectory)\ - default.rd.xml 2.2.0-beta2-build3300 $(FrameworkLibPath)\..\tools\ - + + + + - + + diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml index eb9c9beaf2c..5ac75e69370 100644 --- a/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml +++ b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml @@ -2,10 +2,6 @@ xmlns="http://schemas.microsoft.com/netfx/2013/01/metadata"> - - - - @@ -30,10 +26,17 @@ - + + + + + + + + \ No newline at end of file From ca4b77477a5c0f61aba20923300ec34f62f62b4c Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Fri, 19 Jan 2018 20:43:41 -0800 Subject: [PATCH 07/16] Add json parsing and xml logs --- tests/CoreFX/build-and-run-test.cmd | 6 ++++-- tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml | 1 - .../src/Common/CoreFXTestUtils/TestArchiveUtils.cs | 10 +--------- .../src/Common/CoreFXTestUtils/XUnitResultFormatter.cs | 9 +++++++++ tests/runtest.cmd | 4 ++-- 5 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 tests/CoreFX/runtest/src/Common/CoreFXTestUtils/XUnitResultFormatter.cs diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index 6a6d0805489..e3439230a96 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -21,6 +21,8 @@ set TestFileName=%2 :: Copy artefacts necessary to compile and run the xunit exe copy /Y "%~dp0\runtest\CoreFXTestHarness\*" "%TestFolder%" +:: Create log dir +if not exist %__LogsDir%\CoreFX md %__LogsDir%\CoreFX :: Workaround until we have a better reflection engine :: Add name of currently executing test to rd.xml @@ -30,7 +32,7 @@ if "%CoreRT_BuildArch%" == "x64" ( call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" ) echo msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=true" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj -call msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=true" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj +call msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj echo %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll \ No newline at end of file +call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -xml %__LogDir%\CoreFX\%TestFileName%.xml \ No newline at end of file diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml index 5ac75e69370..73e2e427a3c 100644 --- a/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml +++ b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml @@ -28,7 +28,6 @@ - diff --git a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs index 195702568a7..35f1b3d8a9a 100644 --- a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs +++ b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs @@ -11,7 +11,6 @@ namespace CoreFXTestUtils.TestArchiveUtils { public static class TestArchiveUtils { - static Dictionary testNames = new Dictionary(){ { "Common.Tests", "" }, { "System.Collections.Tests", "" }, { "asdasdasd", "" } }; private static HttpClient httpClient; private static bool cleanTestBuild = false; @@ -21,13 +20,6 @@ public static void Main(string[] args) string testUrl = string.Empty; string testListPath = string.Empty; - Console.WriteLine("Passed " + args.Length + " arguments; which are"); - - foreach (string arg in args) - { - Console.WriteLine(arg); - } - for (int i = 0; i < args.Length; i++) { switch (args[i]) @@ -67,7 +59,7 @@ public static void Main(string[] args) Directory.CreateDirectory(outputDir); } // parse args - SetupTests(testNames, testUrl, outputDir).Wait(); + SetupTests(ReadTestNames(testListPath), testUrl, outputDir).Wait(); } public static Dictionary ReadTestNames(string testFilePath) diff --git a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/XUnitResultFormatter.cs b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/XUnitResultFormatter.cs new file mode 100644 index 00000000000..73e7796ac6b --- /dev/null +++ b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/XUnitResultFormatter.cs @@ -0,0 +1,9 @@ + + +namespace CoreFXTestUtils +{ + public class XUnitResultFormatter + { + } + +} diff --git a/tests/runtest.cmd b/tests/runtest.cmd index f253b20f34e..ffd4302c4a6 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -422,7 +422,7 @@ goto :eof set CLRCustomTestLauncher=%CoreRT_TestRoot%\CoreCLR\build-and-run-test.cmd set XunitTestBinBase=!CoreRT_TestExtRepo_CoreCLR! pushd %CoreRT_TestRoot%\CoreCLR\runtest - + "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore /p:RepoLocalBuild=true src\TestWrappersConfig\XUnitTooling.depproj if errorlevel 1 ( exit /b 1 @@ -451,7 +451,6 @@ goto :eof :TestExtRepoCoreFX - ::if "%CoreRT_TestExtRepo_CoreFX%" == "" ( set CoreRT_TestExtRepo_CoreFX=%CoreRT_TestRoot%\..\tests_downloaded\CoreFX @@ -463,6 +462,7 @@ goto :eof set FXCustomTestLauncher=%CoreRT_TestRoot%\CoreFX\build-and-run-test.cmd set XunitTestBinBase=!CoreRT_TestExtRepo_CoreFX! + set XunitLog=!CoreRT_TestExtRepo_CoreFX!\logs pushd %CoreRT_TestRoot%\CoreFX\runtest "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore /p:RepoLocalBuild=true src\TestWrappersConfig\XUnitTooling.depproj From 16ec04b4eb2a9246c2772b3bae2f886c47180d95 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Mon, 22 Jan 2018 17:49:03 -0800 Subject: [PATCH 08/16] End-to-end test run and display --- tests/CoreFX/build-and-run-test.cmd | 10 +- tests/CoreFX/runtest/runtest.cmd | 3 +- .../CoreFXTestUtils/XUnitResultFormatter.cs | 9 -- .../CoreFX.TestUtils.TestFileSetup.csproj} | 0 .../TestFileSetup/TestFileSetup.cs} | 19 +-- .../XUnit/CoreFX.TestUtils.XUnit.csproj | 12 ++ .../src/TestUtils/XUnit/ResultFormatter.cs | 109 ++++++++++++++++++ tests/runtest.cmd | 51 +++++--- 8 files changed, 179 insertions(+), 34 deletions(-) delete mode 100644 tests/CoreFX/runtest/src/Common/CoreFXTestUtils/XUnitResultFormatter.cs rename tests/CoreFX/runtest/src/{Common/CoreFXTestUtils/CoreFXTestUtils.TestArchiveUtils.csproj => TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj} (100%) rename tests/CoreFX/runtest/src/{Common/CoreFXTestUtils/TestArchiveUtils.cs => TestUtils/TestFileSetup/TestFileSetup.cs} (92%) create mode 100644 tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj create mode 100644 tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index e3439230a96..64c64c7169f 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -21,9 +21,13 @@ set TestFileName=%2 :: Copy artefacts necessary to compile and run the xunit exe copy /Y "%~dp0\runtest\CoreFXTestHarness\*" "%TestFolder%" -:: Create log dir -if not exist %__LogsDir%\CoreFX md %__LogsDir%\CoreFX +:: Create log dir if it doesn't exist +if not exist %XunitLogDir% md %XunitLogDir% +if not exist %TestFolder%\%TestExecutable%.exe ( + :: Not a test we support yet, exit silently + exit /b 0 +) :: Workaround until we have a better reflection engine :: Add name of currently executing test to rd.xml powershell -Command "(Get-Content %TestFolder%\default.rd.xml).replace('*Application*', '%TestFileName%') | Set-Content %TestFolder%\default.rd.xml" @@ -35,4 +39,4 @@ echo msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_Toolchain call msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj echo %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -xml %__LogDir%\CoreFX\%TestFileName%.xml \ No newline at end of file +call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -xml %XunitLogDir%\%TestFileName%.xml \ No newline at end of file diff --git a/tests/CoreFX/runtest/runtest.cmd b/tests/CoreFX/runtest/runtest.cmd index 237a835654e..8f42a4db8f4 100644 --- a/tests/CoreFX/runtest/runtest.cmd +++ b/tests/CoreFX/runtest/runtest.cmd @@ -55,7 +55,7 @@ shift set "__TestWorkingDir=%CoreRT_TestRoot\CoreFX%" -if not defined XunitTestBinBase set XunitTestBinBase=%__TestWorkingDir% +if not defined XunitTestBinBase set XunitTestBinBase=%__TestWorkingDir% if not exist %__LogsDir% md %__LogsDir% @@ -94,6 +94,7 @@ if not defined __Sequential ( for /D %%i in ("%XunitTestBinBase%\*" ) do ( set TestFolderName=%%i set TestFileName=%%~nxi + echo %FXCustomTestLauncher% !TestFolderName! !TestFileName! call %FXCustomTestLauncher% !TestFolderName! !TestFileName! ) \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/XUnitResultFormatter.cs b/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/XUnitResultFormatter.cs deleted file mode 100644 index 73e7796ac6b..00000000000 --- a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/XUnitResultFormatter.cs +++ /dev/null @@ -1,9 +0,0 @@ - - -namespace CoreFXTestUtils -{ - public class XUnitResultFormatter - { - } - -} diff --git a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.TestArchiveUtils.csproj b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj similarity index 100% rename from tests/CoreFX/runtest/src/Common/CoreFXTestUtils/CoreFXTestUtils.TestArchiveUtils.csproj rename to tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj diff --git a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs similarity index 92% rename from tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs rename to tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs index 35f1b3d8a9a..1975e0a4083 100644 --- a/tests/CoreFX/runtest/src/Common/CoreFXTestUtils/TestArchiveUtils.cs +++ b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs @@ -7,9 +7,9 @@ using System.Threading.Tasks; using Newtonsoft.Json; -namespace CoreFXTestUtils.TestArchiveUtils +namespace CoreFX.TestUtils.TestFileSetup { - public static class TestArchiveUtils + public static class TestFileSetup { private static HttpClient httpClient; private static bool cleanTestBuild = false; @@ -59,7 +59,7 @@ public static void Main(string[] args) Directory.CreateDirectory(outputDir); } // parse args - SetupTests(ReadTestNames(testListPath), testUrl, outputDir).Wait(); + SetupTests(testUrl, outputDir, ReadTestNames(testListPath)).Wait(); } public static Dictionary ReadTestNames(string testFilePath) @@ -67,7 +67,8 @@ public static Dictionary ReadTestNames(string testFilePath) Debug.Assert(File.Exists(testFilePath)); Dictionary testNames = new Dictionary(); - // We assume we're being a passed a list of test assembly names, so ignore anything that's not a string + + // We're being a passed a list of test assembly names, so anything that's not a string is invalid using (var sr = new StreamReader(testFilePath)) using (var jsonReader = new JsonTextReader(sr)) { @@ -83,7 +84,7 @@ public static Dictionary ReadTestNames(string testFilePath) return testNames; } - public static async Task SetupTests(Dictionary testNames, string jsonUrl, string destinationDirectory) + public static async Task SetupTests(string jsonUrl, string destinationDirectory, Dictionary testNames = null) { Debug.Assert(Directory.Exists(destinationDirectory)); @@ -93,20 +94,22 @@ public static async Task SetupTests(Dictionary testNames, string Directory.CreateDirectory(tempDirPath); } - Dictionary testPayloads = await GetTestUrls(testNames, jsonUrl); + Dictionary testPayloads = await GetTestUrls(jsonUrl, testNames); await GetTestArchives(testPayloads, tempDirPath); ExpandArchivesInDirectory(tempDirPath, destinationDirectory); Directory.Delete(tempDirPath); } - public static async Task> GetTestUrls(Dictionary testNames, string jsonUrl) + public static async Task> GetTestUrls(string jsonUrl, Dictionary testNames = null) { if(httpClient is null) { httpClient = new HttpClient(); } + bool allTests = testNames == null; + // Set up the json stream reader using (var responseStream = await httpClient.GetStreamAsync(jsonUrl)) using (var streamReader = new StreamReader(responseStream)) @@ -131,7 +134,7 @@ public static async Task> GetTestUrls(Dictionary + + netcoreapp2.0 + Exe + + + + + + + + \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs new file mode 100644 index 00000000000..e3f71d84693 --- /dev/null +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Net.Http; +using System.Reflection; +using System.Threading.Tasks; +using System.Xml; +using Newtonsoft.Json; + +namespace CoreFX.TestUtils.XUnit +{ + public static class ResultFormatter + { + private static string logDir; + private static string logPattern; + + public static void Main(string[] args) + { + ArgumentSyntax syntax = ParseCommandLine(args); + IEnumerable logFiles = DiscoverLogs(logDir, logPattern); + PrintTotals(logFiles); + + } + + private static void PrintTotals(IEnumerable logFiles) + { + int total = 0; + int passed = 0; + int failed = 0; + int skipped = 0; + ulong timeElapsed = 0; + + foreach(string logFile in logFiles) + { + using (XmlReader reader = XmlReader.Create(logFile)) + { + reader.MoveToContent(); + reader.ReadToDescendant("collection"); + do + { + + + // Get total tests in current element + string totalAttr = reader.GetAttribute("total"); + int currentTotal; + Int32.TryParse(totalAttr, out currentTotal); + total += currentTotal; + + // Get passed tests + string passedAttr = reader.GetAttribute("passed"); + int currentPassed; + Int32.TryParse(passedAttr, out currentPassed); + passed += currentPassed; + + // Get failed tests + string failedAttr = reader.GetAttribute("failed"); + int currentFailed; + Int32.TryParse(failedAttr, out currentFailed); + failed += currentFailed; + + // Get skipped tests + string skippedAttr = reader.GetAttribute("skipped"); + int currentSkipped; + Int32.TryParse(skippedAttr, out currentSkipped); + skipped += currentSkipped; + + // Get time elapsed + string timeAttr = reader.GetAttribute("time"); + ulong currentTime; + UInt64.TryParse(timeAttr, out currentTime); + timeElapsed += currentTime; + + } while (reader.ReadToNextSibling("collection")); + + } + } + + Console.WriteLine("=== CoreFX TEST EXECUTION SUMMARY ===: "); + Console.WriteLine(String.Format("Total: {0}, Errors: {1}, Failed: {2}, Skipped: {3}", total, passed, failed, timeElapsed)); + + } + + private static ArgumentSyntax ParseCommandLine(string[] args) + { + AssemblyName name = typeof(ResultFormatter).GetTypeInfo().Assembly.GetName(); + + ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax => + { + syntax.DefineOption("log|logDirectory|logDir", ref logDir, "Path to directory of xml test results"); + syntax.DefineOption("pattern|p", ref logPattern, "Pattern of XUnit log filenames for which to search"); + }); + + return argSyntax; + } + + public static IEnumerable DiscoverLogs(string logDirectory, string logPattern) + { + Debug.Assert(Directory.Exists(logDirectory)); + var logFiles = Directory.EnumerateFiles(logDirectory, logPattern, SearchOption.AllDirectories); + + return logFiles; + } + + + } +} diff --git a/tests/runtest.cmd b/tests/runtest.cmd index ffd4302c4a6..77aab3dd531 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -111,7 +111,7 @@ if NOT "%CoreRT_MultiFileConfiguration%" == "" ( if not exist %__CoreRTTestBinDir%\%CoreRT_MultiFileConfiguration%\ mkdir %__CoreRTTestBinDir%\%CoreRT_MultiFileConfiguration% ) -set __LogDir=%CoreRT_TestRoot%\..\bin\Logs\%__BuildStr%\tests +set __LogDir=%CoreRT_TestRoot%..\bin\Logs\%__BuildStr%\tests if defined VisualStudioVersion goto :RunVCVars @@ -338,15 +338,20 @@ goto :eof exit /b -1 :RestoreCoreFXTests - + :: Explicitly restore the test helper project - "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "!CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\CoreFXTestUtils.TestArchiveUtils.csproj" + "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "%CoreRT_TestFileHelperProjectPath%" + "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "%CoreRT_XunitHelperProjectPath%" if errorlevel 1 ( exit /b 1 ) :: Build the test helper project - "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:RepoLocalBuild=true "!CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\CoreFXTestUtils.TestArchiveUtils.csproj" + echo "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_TestFileHelperProjectPath%" + echo "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_XunitHelperProjectPath%" + "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_TestFileHelperProjectPath%" + "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_XunitHelperProjectPath%" + if errorlevel 1 ( exit /b 1 ) @@ -360,7 +365,14 @@ goto :eof set /p TESTS_REMOTE_URL=< "%~dp0/CoreFXTestListURL.txt" set TEST_LIST="%~dp0/Top200.CoreFX.issues.json" - "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestRoot!\CoreFX\runtest\src\Common\CoreFXTestUtils\bin\!CoreRT_BuildArch!\!CoreRT_BuildType!\netcoreapp2.0\CoreFXTestUtils.TestArchiveUtils.dll --clean --outputDirectory !CoreRT_TestExtRepo_CoreFX! --testListJsonPath "%TEST_LIST%" --testUrl "%TESTS_REMOTE_URL%" + + if not exist !CoreRT_TestingUtilitiesOutputDir!\%CoreRT_TestFileHelperName%.dll ( + echo File !CoreRT_TestingUtilitiesOutputDir!\%CoreRT_TestFileHelperName%.dll not found. + exit /b 1 + ) + + "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestingUtilitiesOutputDir!\%CoreRT_TestFileHelperName%.dll --clean --outputDirectory !CoreRT_TestExtRepo_CoreFX! --testListJsonPath "%TEST_LIST%" --testUrl "%TESTS_REMOTE_URL%" + exit /b 0 @@ -451,8 +463,18 @@ goto :eof :TestExtRepoCoreFX + set CoreRT_TestExtRepo_CoreFX=%CoreRT_TestRoot%\..\tests_downloaded\CoreFX + set CoreRT_TestingUtilitiesOutputDir=%CoreRT_TestExtRepo_CoreFX%\Utilities + + :: Set paths to helpers + set CoreRT_TestFileHelperName=CoreFX.TestUtils.TestFileSetup + set CoreRT_TestFileHelperProjectPath="%CoreRT_TestRoot%\CoreFX\runtest\src\TestUtils\TestFileSetup\%CoreRT_TestFileHelperName%.csproj" + + set CoreRT_XunitHelperName=CoreFX.TestUtils.XUnit + set CoreRT_XunitHelperProjectPath="%CoreRT_TestRoot%\CoreFX\runtest\src\TestUtils\XUnit\%CoreRT_XunitHelperName%.csproj" + + ::if "%CoreRT_TestExtRepo_CoreFX%" == "" ( - set CoreRT_TestExtRepo_CoreFX=%CoreRT_TestRoot%\..\tests_downloaded\CoreFX call :RestoreCoreFXTests if errorlevel 1 ( @@ -461,17 +483,20 @@ goto :eof ::) set FXCustomTestLauncher=%CoreRT_TestRoot%\CoreFX\build-and-run-test.cmd - set XunitTestBinBase=!CoreRT_TestExtRepo_CoreFX! - set XunitLog=!CoreRT_TestExtRepo_CoreFX!\logs + set XunitTestBinBase=%CoreRT_TestExtRepo_CoreFX% + set XunitLogDir= %__LogDir%\CoreFX pushd %CoreRT_TestRoot%\CoreFX\runtest - "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore /p:RepoLocalBuild=true src\TestWrappersConfig\XUnitTooling.depproj + rem TODO Add single test/target test support; add exclude tests + + + + echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% + call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% if errorlevel 1 ( exit /b 1 ) - rem TODO Add single test/target test support; add exclude tests - - echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% - call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %__LogDir% + echo "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestingUtilitiesOutputDir!\!CoreRT_XunitHelperName!.dll --logDir "%XunitLogDir%" --pattern "*.xml" + "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestingUtilitiesOutputDir!\!CoreRT_XunitHelperName!.dll --logDir "%XunitLogDir%" --pattern "*.xml" \ No newline at end of file From 1d7e4209c4159ab3166e161ad43949238dad2047 Mon Sep 17 00:00:00 2001 From: A-And Date: Tue, 23 Jan 2018 07:08:20 -0800 Subject: [PATCH 09/16] Helper project style changes --- .../CoreFX.TestUtils.TestFileSetup.csproj | 3 +- .../TestUtils/TestFileSetup/TestFileSetup.cs | 57 +++++++------------ .../XUnit/CoreFX.TestUtils.XUnit.csproj | 2 - .../src/TestUtils/XUnit/ResultFormatter.cs | 1 - 4 files changed, 20 insertions(+), 43 deletions(-) diff --git a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj index 0d4d803c1af..e616435a9d3 100644 --- a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj +++ b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj @@ -5,7 +5,6 @@ + - - \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs index 1975e0a4083..589556fef7a 100644 --- a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs +++ b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.CommandLine; using System.Diagnostics; using System.IO; using System.IO.Compression; @@ -14,46 +15,13 @@ public static class TestFileSetup private static HttpClient httpClient; private static bool cleanTestBuild = false; + private static string outputDir; + private static string testUrl; + private static string testListPath; + public static void Main(string[] args) { - string outputDir = string.Empty; - string testUrl = string.Empty; - string testListPath = string.Empty; - - for (int i = 0; i < args.Length; i++) - { - switch (args[i]) - { - case "--outputDirectory": - case "--out": - case "--outputDir": - if(i + 1 < args.Length && !args[i+1].Substring(0,2).Equals("--")) - { - outputDir = args[i + 1]; - i++; - } - break; - case "--testUrl": - if (i + 1 < args.Length && !args[i + 1].Substring(0, 2).Equals("--")) - { - testUrl = args[i + 1]; - i++; - } - break; - case "--testListJsonPath": - if (i + 1 < args.Length && !args[i + 1].Substring(0, 2).Equals("--")) - { - testListPath = args[i + 1]; - i++; - } - break; - case "--clean": - cleanTestBuild = true; - break; - - } - } - + ArgumentSyntax argSyntax = ParseCommandLine(args); if (!Directory.Exists(outputDir)) { Directory.CreateDirectory(outputDir); @@ -62,6 +30,19 @@ public static void Main(string[] args) SetupTests(testUrl, outputDir, ReadTestNames(testListPath)).Wait(); } + public static ArgumentSyntax ParseCommandLine(string[] args) + { + ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax => + { + syntax.DefineOption("out|outDir|outputDirectory", ref outputDir, "Directory where tests are downloaded"); + syntax.DefineOption("testUrl", ref testUrl, "URL, pointing to the list of tests"); + syntax.DefineOption("testListJsonPath", ref testListPath, "JSON-formatted list of test assembly names to download"); + syntax.DefineOption("clean", ref cleanTestBuild, "Remove all previously built test assemblies"); + }); + + return argSyntax; + } + public static Dictionary ReadTestNames(string testFilePath) { Debug.Assert(File.Exists(testFilePath)); diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj b/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj index 90acdef9a30..e616435a9d3 100644 --- a/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj @@ -7,6 +7,4 @@ - - \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs index e3f71d84693..d2545040991 100644 --- a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs @@ -85,7 +85,6 @@ private static void PrintTotals(IEnumerable logFiles) private static ArgumentSyntax ParseCommandLine(string[] args) { - AssemblyName name = typeof(ResultFormatter).GetTypeInfo().Assembly.GetName(); ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax => { From b4c1b7c0da14afdb534fee0e847bf02cf286007b Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Tue, 23 Jan 2018 18:21:07 -0800 Subject: [PATCH 10/16] Remove direct dll references add package dependencies --- tests/CoreFX/build-and-run-test.cmd | 10 ++-- tests/CoreFX/dependencies.props | 0 .../runtest/CoreFXTestHarness/Test.csproj | 53 +++++++++++-------- .../src/TestUtils/XUnit/ResultFormatter.cs | 5 +- tests/runtest.cmd | 5 -- 5 files changed, 36 insertions(+), 37 deletions(-) create mode 100644 tests/CoreFX/dependencies.props diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index 64c64c7169f..d7e271b4bde 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -19,7 +19,7 @@ set TestFileName=%2 :: Copy artefacts necessary to compile and run the xunit exe -copy /Y "%~dp0\runtest\CoreFXTestHarness\*" "%TestFolder%" +copy /Y "%~dp0\runtest\CoreFXTestHarness\*" "%TestFolder%" >nul :: Create log dir if it doesn't exist if not exist %XunitLogDir% md %XunitLogDir% @@ -33,10 +33,10 @@ if not exist %TestFolder%\%TestExecutable%.exe ( powershell -Command "(Get-Content %TestFolder%\default.rd.xml).replace('*Application*', '%TestFileName%') | Set-Content %TestFolder%\default.rd.xml" if "%CoreRT_BuildArch%" == "x64" ( - call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" + call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" >nul ) -echo msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=true" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj -call msbuild /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj -echo %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll +call "%CoreRT_CliDir%\dotnet.exe" publish /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj +REM /verbosity:diag > "C:\Users\anandono\source\repos\runtestFX.log" + call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -xml %XunitLogDir%\%TestFileName%.xml \ No newline at end of file diff --git a/tests/CoreFX/dependencies.props b/tests/CoreFX/dependencies.props new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj index 37d551df945..94c5a24a978 100644 --- a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj +++ b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj @@ -1,52 +1,59 @@ - + + + $(ExecutableName) .exe Exe - $(MSBuildProjectDirectory)\ - $(MSBuildProjectDirectory)\ + + + 2.2.0-beta2-build3300 $(FrameworkLibPath)\..\tools\ + netcoreapp2.0 + + $(NoWarn);NU1603 + - + + 1.0.1-prerelease-02104-02 + + - - - - - - - + + + + $(MSBuildProjectDirectory)\ + $(MSBuildProjectDirectory)\ + + - - - \ No newline at end of file + \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs index d2545040991..88a8b41e058 100644 --- a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs @@ -22,7 +22,6 @@ public static void Main(string[] args) ArgumentSyntax syntax = ParseCommandLine(args); IEnumerable logFiles = DiscoverLogs(logDir, logPattern); PrintTotals(logFiles); - } private static void PrintTotals(IEnumerable logFiles) @@ -41,8 +40,6 @@ private static void PrintTotals(IEnumerable logFiles) reader.ReadToDescendant("collection"); do { - - // Get total tests in current element string totalAttr = reader.GetAttribute("total"); int currentTotal; @@ -79,7 +76,7 @@ private static void PrintTotals(IEnumerable logFiles) } Console.WriteLine("=== CoreFX TEST EXECUTION SUMMARY ===: "); - Console.WriteLine(String.Format("Total: {0}, Errors: {1}, Failed: {2}, Skipped: {3}", total, passed, failed, timeElapsed)); + Console.WriteLine(String.Format("Total: {0}, Passed: {1}, Failed: {2}, Skipped: {3}", total, passed, failed, timeElapsed)); } diff --git a/tests/runtest.cmd b/tests/runtest.cmd index 77aab3dd531..058d50c3a31 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -347,8 +347,6 @@ goto :eof ) :: Build the test helper project - echo "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_TestFileHelperProjectPath%" - echo "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_XunitHelperProjectPath%" "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_TestFileHelperProjectPath%" "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_XunitHelperProjectPath%" @@ -488,8 +486,6 @@ goto :eof pushd %CoreRT_TestRoot%\CoreFX\runtest rem TODO Add single test/target test support; add exclude tests - - echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% @@ -497,6 +493,5 @@ goto :eof exit /b 1 ) - echo "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestingUtilitiesOutputDir!\!CoreRT_XunitHelperName!.dll --logDir "%XunitLogDir%" --pattern "*.xml" "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestingUtilitiesOutputDir!\!CoreRT_XunitHelperName!.dll --logDir "%XunitLogDir%" --pattern "*.xml" \ No newline at end of file From d0a6b8a7359fc3e97924e1122701c5f56150c260 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Wed, 24 Jan 2018 14:51:55 -0800 Subject: [PATCH 11/16] Stylistic changes + deps --- tests/CoreFX/build-and-run-test.cmd | 4 ++ tests/CoreFX/dependencies.props | 65 +++++++++++++++++++ tests/CoreFX/runtest.proj | 0 .../runtest/CoreFXTestHarness/Test.csproj | 13 +++- tests/CoreFX/runtest/dir.props | 29 ++------- .../CoreFX.TestUtils.TestFileSetup.csproj | 9 ++- .../XUnit/CoreFX.TestUtils.XUnit.csproj | 9 ++- .../src/TestUtils/XUnit/ResultFormatter.cs | 1 + tests/Top200.CoreFX.issues.json | 4 -- tests/TopN.CoreFX.issues.json | 5 ++ tests/runtest.cmd | 17 ++--- 11 files changed, 111 insertions(+), 45 deletions(-) delete mode 100644 tests/CoreFX/runtest.proj delete mode 100644 tests/Top200.CoreFX.issues.json create mode 100644 tests/TopN.CoreFX.issues.json diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index d7e271b4bde..93d1bc6c5e9 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -36,7 +36,11 @@ if "%CoreRT_BuildArch%" == "x64" ( call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" >nul ) +echo Build %TestFileName% + call "%CoreRT_CliDir%\dotnet.exe" publish /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj REM /verbosity:diag > "C:\Users\anandono\source\repos\runtestFX.log" +echo Executing %TestFileName% + call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -xml %XunitLogDir%\%TestFileName%.xml \ No newline at end of file diff --git a/tests/CoreFX/dependencies.props b/tests/CoreFX/dependencies.props index e69de29bb2d..eb666faa8f6 100644 --- a/tests/CoreFX/dependencies.props +++ b/tests/CoreFX/dependencies.props @@ -0,0 +1,65 @@ + + + + + + 9f9f1cf6e6c4a969e8d0158936427b9896300035 + 9f9f1cf6e6c4a969e8d0158936427b9896300035 + 9f9f1cf6e6c4a969e8d0158936427b9896300035 + + + + + beta-24530-02 + beta-24523-00 + + + + + 1.2.0-beta-24529-03 + 2.2.0-beta2-build3300 + + + + + 0.1.0-e160909-1 + 10.0.1 + + + + + + $(BaseDotNetBuildInfo)corefx/$(DependencyBranch) + $(CoreFxCurrentRef) + + + $(BaseDotNetBuildInfo)coreclr/$(DependencyBranch) + $(CoreClrCurrentRef) + + + $(BaseDotNetBuildInfo)projectk-tfs/$(DependencyBranch) + $(ExternalCurrentRef) + + + + https://raw.githubusercontent.com/dotnet/versions + + + + $(MSBuildThisFileFullPath) + CoreFxExpectedPrerelease + CoreFx + + + $(MSBuildThisFileFullPath) + CoreClrPackageVersion + Microsoft.NETCore.Runtime.CoreCLR + + + $(MSBuildThisFileFullPath) + ExternalExpectedPrerelease + External + + + + diff --git a/tests/CoreFX/runtest.proj b/tests/CoreFX/runtest.proj deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj index 94c5a24a978..c59e1e9361f 100644 --- a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj +++ b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj @@ -1,5 +1,6 @@ + $(ExecutableName) @@ -8,7 +9,10 @@ + 2.2.0-beta2-build3300 + 2.0.1-rc2 + 1.0.1-prerelease-02104-02 $(FrameworkLibPath)\..\tools\ netcoreapp2.0 @@ -20,7 +24,7 @@ - 2.0.1-rc2 + $(XunitAbstractionsVersion) $(XunitPackageVersion) @@ -38,22 +42,27 @@ $(XunitPackageVersion) - 1.0.1-prerelease-02104-02 + $(XunitNetcoreExtensionsVersion) + + $(MSBuildProjectDirectory)\ $(MSBuildProjectDirectory)\ + + + \ No newline at end of file diff --git a/tests/CoreFX/runtest/dir.props b/tests/CoreFX/runtest/dir.props index 63c226ff273..e947c5a9968 100644 --- a/tests/CoreFX/runtest/dir.props +++ b/tests/CoreFX/runtest/dir.props @@ -1,5 +1,7 @@ - + + + @@ -7,12 +9,6 @@ $(OS) - - - 1.0.0-rc3-20150510-01 - Microsoft.Net.ToolsetCompilers - - false @@ -23,7 +19,7 @@ $(ToolsDir)dotnetcli/ true - + @@ -34,21 +30,4 @@ - - - false - - - - $(DotnetCliPath)dotnet.exe - $(DotnetCliPath)dotnet - - @(DnuSourceList -> '--source %(Identity)', ' ') - @(DnuRestoreDir -> '%(Identity)', ' ') - - $(DnuRestoreCommand) "$(DotnetToolCommand)" - $(DnuRestoreCommand) restore - $(DnuRestoreCommand) --packages "$(PackagesDir.TrimEnd('/\'.ToCharArray()))" $(DnuRestoreSource) - - diff --git a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj index e616435a9d3..adc435e32a1 100644 --- a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj +++ b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj @@ -1,10 +1,15 @@ + netcoreapp2.0 Exe - - + + $(NewtonsoftJsonVersion) + + + $(SystemCommandLineVersion) + \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj b/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj index e616435a9d3..adc435e32a1 100644 --- a/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj @@ -1,10 +1,15 @@ + netcoreapp2.0 Exe - - + + $(NewtonsoftJsonVersion) + + + $(SystemCommandLineVersion) + \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs index 88a8b41e058..501782bda4c 100644 --- a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs @@ -77,6 +77,7 @@ private static void PrintTotals(IEnumerable logFiles) Console.WriteLine("=== CoreFX TEST EXECUTION SUMMARY ===: "); Console.WriteLine(String.Format("Total: {0}, Passed: {1}, Failed: {2}, Skipped: {3}", total, passed, failed, timeElapsed)); + Console.WriteLine("Detailed logs written to: " + logDir); } diff --git a/tests/Top200.CoreFX.issues.json b/tests/Top200.CoreFX.issues.json deleted file mode 100644 index 5200c64af57..00000000000 --- a/tests/Top200.CoreFX.issues.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - "System.Collections.Tests", - "System.Collections.Immutable.Tests" -] \ No newline at end of file diff --git a/tests/TopN.CoreFX.issues.json b/tests/TopN.CoreFX.issues.json new file mode 100644 index 00000000000..40675856912 --- /dev/null +++ b/tests/TopN.CoreFX.issues.json @@ -0,0 +1,5 @@ +[ + "System.Collections.Tests", + "System.Collections.Immutable.Tests", + "System.Net.Security.Tests" +] \ No newline at end of file diff --git a/tests/runtest.cmd b/tests/runtest.cmd index 058d50c3a31..415240815f7 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -362,7 +362,7 @@ goto :eof ) set /p TESTS_REMOTE_URL=< "%~dp0/CoreFXTestListURL.txt" - set TEST_LIST="%~dp0/Top200.CoreFX.issues.json" + set TEST_LIST="%~dp0/TopN.CoreFX.issues.json" if not exist !CoreRT_TestingUtilitiesOutputDir!\%CoreRT_TestFileHelperName%.dll ( echo File !CoreRT_TestingUtilitiesOutputDir!\%CoreRT_TestFileHelperName%.dll not found. @@ -471,21 +471,18 @@ goto :eof set CoreRT_XunitHelperName=CoreFX.TestUtils.XUnit set CoreRT_XunitHelperProjectPath="%CoreRT_TestRoot%\CoreFX\runtest\src\TestUtils\XUnit\%CoreRT_XunitHelperName%.csproj" - - ::if "%CoreRT_TestExtRepo_CoreFX%" == "" ( - - call :RestoreCoreFXTests - if errorlevel 1 ( - exit /b 1 - ) - ::) + :: TODO Check if each requested test has already been restored + call :RestoreCoreFXTests + if errorlevel 1 ( + exit /b 1 + ) set FXCustomTestLauncher=%CoreRT_TestRoot%\CoreFX\build-and-run-test.cmd set XunitTestBinBase=%CoreRT_TestExtRepo_CoreFX% set XunitLogDir= %__LogDir%\CoreFX pushd %CoreRT_TestRoot%\CoreFX\runtest - rem TODO Add single test/target test support; add exclude tests + :: TODO Add single test/target test support; add exclude tests argument echo runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% call runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% From ca7fb8215915fc5668dbb7be9042979c9d1cf0a2 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Wed, 24 Jan 2018 16:13:03 -0800 Subject: [PATCH 12/16] Style fixup --- tests/CoreFX/build-and-run-test.cmd | 3 +- tests/CoreFX/dependencies.props | 122 +++++++++--------- .../runtest/CoreFXTestHarness/Test.csproj | 7 +- .../runtest/CoreFXTestHarness/default.rd.xml | 2 +- tests/CoreFX/runtest/runtest.cmd | 2 +- .../CoreFX.TestUtils.TestFileSetup.csproj | 2 +- .../XUnit/CoreFX.TestUtils.XUnit.csproj | 2 +- .../TestWrappersConfig/XUnitTooling.depproj | 62 --------- tests/TopN.CoreFX.issues.json | 2 +- tests/runtest.cmd | 2 +- 10 files changed, 68 insertions(+), 138 deletions(-) delete mode 100644 tests/CoreFX/runtest/src/TestWrappersConfig/XUnitTooling.depproj diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index 93d1bc6c5e9..b18f63c09dc 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -38,8 +38,7 @@ if "%CoreRT_BuildArch%" == "x64" ( echo Build %TestFileName% -call "%CoreRT_CliDir%\dotnet.exe" publish /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:PackagesDir=%~dp0..\..\packages\ /p:ExecutableName=%TestExecutable% %TestFolder%\Test.csproj -REM /verbosity:diag > "C:\Users\anandono\source\repos\runtestFX.log" +call "%CoreRT_CliDir%\dotnet.exe" publish %TestFolder%\Test.csproj /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:TestRootDir=%~dp0 /p:ExecutableName=%TestExecutable% echo Executing %TestFileName% diff --git a/tests/CoreFX/dependencies.props b/tests/CoreFX/dependencies.props index eb666faa8f6..18e25734374 100644 --- a/tests/CoreFX/dependencies.props +++ b/tests/CoreFX/dependencies.props @@ -1,65 +1,59 @@ - - - - - - 9f9f1cf6e6c4a969e8d0158936427b9896300035 - 9f9f1cf6e6c4a969e8d0158936427b9896300035 - 9f9f1cf6e6c4a969e8d0158936427b9896300035 - - - - - beta-24530-02 - beta-24523-00 - - - - - 1.2.0-beta-24529-03 - 2.2.0-beta2-build3300 - - - - - 0.1.0-e160909-1 - 10.0.1 - - - - - - $(BaseDotNetBuildInfo)corefx/$(DependencyBranch) - $(CoreFxCurrentRef) - - - $(BaseDotNetBuildInfo)coreclr/$(DependencyBranch) - $(CoreClrCurrentRef) - - - $(BaseDotNetBuildInfo)projectk-tfs/$(DependencyBranch) - $(ExternalCurrentRef) - - - - https://raw.githubusercontent.com/dotnet/versions - - - - $(MSBuildThisFileFullPath) - CoreFxExpectedPrerelease - CoreFx - - - $(MSBuildThisFileFullPath) - CoreClrPackageVersion - Microsoft.NETCore.Runtime.CoreCLR - - - $(MSBuildThisFileFullPath) - ExternalExpectedPrerelease - External - - - + + + + + 9f9f1cf6e6c4a969e8d0158936427b9896300035 + 9f9f1cf6e6c4a969e8d0158936427b9896300035 + 9f9f1cf6e6c4a969e8d0158936427b9896300035 + + + + beta-24530-02 + beta-24523-00 + + + + 1.2.0-beta-24529-03 + 2.2.0-beta2-build3300 + + + + 0.1.0-e160909-1 + 10.0.1 + 2.0.1-rc2 + 1.0.1-prerelease-02104-02 + + + + $(BaseDotNetBuildInfo)corefx/$(DependencyBranch) + $(CoreFxCurrentRef) + + + $(BaseDotNetBuildInfo)coreclr/$(DependencyBranch) + $(CoreClrCurrentRef) + + + $(BaseDotNetBuildInfo)projectk-tfs/$(DependencyBranch) + $(ExternalCurrentRef) + + + https://raw.githubusercontent.com/dotnet/versions + + + $(MSBuildThisFileFullPath) + CoreFxExpectedPrerelease + CoreFx + + + $(MSBuildThisFileFullPath) + CoreClrPackageVersion + Microsoft.NETCore.Runtime.CoreCLR + + + $(MSBuildThisFileFullPath) + ExternalExpectedPrerelease + External + + diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj index c59e1e9361f..8c3d2d8aaad 100644 --- a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj +++ b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj @@ -1,6 +1,8 @@ + + $(ExecutableName) @@ -10,9 +12,6 @@ - 2.2.0-beta2-build3300 - 2.0.1-rc2 - 1.0.1-prerelease-02104-02 $(FrameworkLibPath)\..\tools\ netcoreapp2.0 @@ -65,4 +64,4 @@ - \ No newline at end of file + diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml index 73e2e427a3c..81d5bc9e0a6 100644 --- a/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml +++ b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml @@ -38,4 +38,4 @@ - \ No newline at end of file + diff --git a/tests/CoreFX/runtest/runtest.cmd b/tests/CoreFX/runtest/runtest.cmd index 8f42a4db8f4..b5f9038e642 100644 --- a/tests/CoreFX/runtest/runtest.cmd +++ b/tests/CoreFX/runtest/runtest.cmd @@ -97,4 +97,4 @@ for /D %%i in ("%XunitTestBinBase%\*" ) do ( echo %FXCustomTestLauncher% !TestFolderName! !TestFileName! call %FXCustomTestLauncher% !TestFolderName! !TestFileName! -) \ No newline at end of file +) diff --git a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj index adc435e32a1..3653a481442 100644 --- a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj +++ b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj @@ -12,4 +12,4 @@ $(SystemCommandLineVersion) - \ No newline at end of file + diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj b/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj index adc435e32a1..3653a481442 100644 --- a/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj @@ -12,4 +12,4 @@ $(SystemCommandLineVersion) - \ No newline at end of file + diff --git a/tests/CoreFX/runtest/src/TestWrappersConfig/XUnitTooling.depproj b/tests/CoreFX/runtest/src/TestWrappersConfig/XUnitTooling.depproj deleted file mode 100644 index 928310a416e..00000000000 --- a/tests/CoreFX/runtest/src/TestWrappersConfig/XUnitTooling.depproj +++ /dev/null @@ -1,62 +0,0 @@ - - - - net46 - 2.2.0-beta2-build3300 - 2.1.0-prerelease-02411-04 - 2.0.1 - - - - - - - - - - - $(XunitPackageVersion) - - - $(XunitPackageVersion) - - - $(XunitPackageVersion) - - - $(XunitPackageVersion) - - - $(XunitAbstractionsVersion) - - - $(XunitPackageVersion) - - - $(XunitPackageVersion) - - - - - - - - - - - - - - - - - diff --git a/tests/TopN.CoreFX.issues.json b/tests/TopN.CoreFX.issues.json index 40675856912..dc5d146b4a8 100644 --- a/tests/TopN.CoreFX.issues.json +++ b/tests/TopN.CoreFX.issues.json @@ -2,4 +2,4 @@ "System.Collections.Tests", "System.Collections.Immutable.Tests", "System.Net.Security.Tests" -] \ No newline at end of file +] diff --git a/tests/runtest.cmd b/tests/runtest.cmd index 415240815f7..322d95f55d2 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -111,7 +111,7 @@ if NOT "%CoreRT_MultiFileConfiguration%" == "" ( if not exist %__CoreRTTestBinDir%\%CoreRT_MultiFileConfiguration%\ mkdir %__CoreRTTestBinDir%\%CoreRT_MultiFileConfiguration% ) -set __LogDir=%CoreRT_TestRoot%..\bin\Logs\%__BuildStr%\tests +set __LogDir=%CoreRT_TestRoot%\..\bin\Logs\%__BuildStr%\tests if defined VisualStudioVersion goto :RunVCVars From 067d611ba8c2e9c2451e56fa096845f81c7e7a1f Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Wed, 24 Jan 2018 16:13:20 -0800 Subject: [PATCH 13/16] XML reader escape --- tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs index 501782bda4c..e0440f5ac8a 100644 --- a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs @@ -34,7 +34,8 @@ private static void PrintTotals(IEnumerable logFiles) foreach(string logFile in logFiles) { - using (XmlReader reader = XmlReader.Create(logFile)) + // TODO XMLReader escapes the character sequence \\.. as just a single backslash \ - Is this intended behavior? + using (XmlReader reader = XmlReader.Create(logFile.Replace(@"\\..", @"\.."))) { reader.MoveToContent(); reader.ReadToDescendant("collection"); @@ -96,8 +97,9 @@ private static ArgumentSyntax ParseCommandLine(string[] args) public static IEnumerable DiscoverLogs(string logDirectory, string logPattern) { Debug.Assert(Directory.Exists(logDirectory)); + Console.WriteLine(logDirectory); var logFiles = Directory.EnumerateFiles(logDirectory, logPattern, SearchOption.AllDirectories); - + return logFiles; } From f39867a38d8c0b8b42c1a050f66d1efc275be0af Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Wed, 24 Jan 2018 16:27:25 -0800 Subject: [PATCH 14/16] Dependency and comment cleanup --- tests/CoreFX/build-and-run-test.cmd | 9 +++--- tests/CoreFX/dependencies.props | 50 ++--------------------------- 2 files changed, 6 insertions(+), 53 deletions(-) diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index b18f63c09dc..045c96565e9 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -1,7 +1,6 @@ -:: Test execution wrapper for CoreCLR tests +:: Test build and execution wrapper for CoreFX tests :: -:: This wrapper is called from each CoreCLR test's .cmd / .sh run script as the custom test launcher. -:: We use this opportunity to invoke the CoreRT compiler and then run the produced native binary. +:: This wrapper is called for each of CoreFX's tests by runtest.cmd :: :: %1 contains test folder :: %2 contains test exe name @@ -12,13 +11,13 @@ setlocal ENABLEDELAYEDEXPANSION set TestFolder=%1 :: -:: We're only interested in referencing the xunit runner - the test dlls will be imported by the test project +:: We're only interested in referencing the xunit runner - the test dlls will be imported by the test wrapper project :: set TestExecutable=xunit.console.netcore set TestFileName=%2 -:: Copy artefacts necessary to compile and run the xunit exe +:: Copy the artefacts we need to compile and run the xunit exe copy /Y "%~dp0\runtest\CoreFXTestHarness\*" "%TestFolder%" >nul :: Create log dir if it doesn't exist diff --git a/tests/CoreFX/dependencies.props b/tests/CoreFX/dependencies.props index 18e25734374..a52c6023fd1 100644 --- a/tests/CoreFX/dependencies.props +++ b/tests/CoreFX/dependencies.props @@ -1,59 +1,13 @@ - - - 9f9f1cf6e6c4a969e8d0158936427b9896300035 - 9f9f1cf6e6c4a969e8d0158936427b9896300035 - 9f9f1cf6e6c4a969e8d0158936427b9896300035 - - - - beta-24530-02 - beta-24523-00 - - - - 1.2.0-beta-24529-03 - 2.2.0-beta2-build3300 - + 0.1.0-e160909-1 10.0.1 + 2.2.0-beta2-build3300 2.0.1-rc2 1.0.1-prerelease-02104-02 - - - $(BaseDotNetBuildInfo)corefx/$(DependencyBranch) - $(CoreFxCurrentRef) - - - $(BaseDotNetBuildInfo)coreclr/$(DependencyBranch) - $(CoreClrCurrentRef) - - - $(BaseDotNetBuildInfo)projectk-tfs/$(DependencyBranch) - $(ExternalCurrentRef) - - - https://raw.githubusercontent.com/dotnet/versions - - - $(MSBuildThisFileFullPath) - CoreFxExpectedPrerelease - CoreFx - - - $(MSBuildThisFileFullPath) - CoreClrPackageVersion - Microsoft.NETCore.Runtime.CoreCLR - - - $(MSBuildThisFileFullPath) - ExternalExpectedPrerelease - External - - From 48d4acca9a9aeac140b4dfc02488ed59550f63ef Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Fri, 26 Jan 2018 18:21:33 -0800 Subject: [PATCH 15/16] Respond to PR feedback --- src/Framework/Framework-uapaot.depproj | 2 ++ tests/CoreFX/build-and-run-test.cmd | 7 +++- .../runtest/CoreFXTestHarness/Test.csproj | 3 -- tests/CoreFX/runtest/runtest.cmd | 34 ++++++++++--------- .../TestUtils/TestFileSetup/TestFileSetup.cs | 11 +++++- .../src/TestUtils/XUnit/ResultFormatter.cs | 9 ++++- tests/runtest.cmd | 10 ++++-- 7 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/Framework/Framework-uapaot.depproj b/src/Framework/Framework-uapaot.depproj index 5ad5937ad5a..ab8e28d15e3 100644 --- a/src/Framework/Framework-uapaot.depproj +++ b/src/Framework/Framework-uapaot.depproj @@ -24,6 +24,8 @@ + + diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd index 045c96565e9..b79323d212f 100644 --- a/tests/CoreFX/build-and-run-test.cmd +++ b/tests/CoreFX/build-and-run-test.cmd @@ -27,6 +27,7 @@ if not exist %TestFolder%\%TestExecutable%.exe ( :: Not a test we support yet, exit silently exit /b 0 ) + :: Workaround until we have a better reflection engine :: Add name of currently executing test to rd.xml powershell -Command "(Get-Content %TestFolder%\default.rd.xml).replace('*Application*', '%TestFileName%') | Set-Content %TestFolder%\default.rd.xml" @@ -37,7 +38,11 @@ if "%CoreRT_BuildArch%" == "x64" ( echo Build %TestFileName% -call "%CoreRT_CliDir%\dotnet.exe" publish %TestFolder%\Test.csproj /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:TestRootDir=%~dp0 /p:ExecutableName=%TestExecutable% +call "%CoreRT_CliDir%\dotnet.exe" publish %TestFolder%\Test.csproj /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:TestRootDir=%~dp0 /p:ExecutableName=%TestExecutable% /nologo +if errorlevel 1 ( + echo Building %TestFileName% failed + exit /b 1 +) echo Executing %TestFileName% diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj index 8c3d2d8aaad..634dba2940e 100644 --- a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj +++ b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj @@ -47,9 +47,6 @@ - - - diff --git a/tests/CoreFX/runtest/runtest.cmd b/tests/CoreFX/runtest/runtest.cmd index b5f9038e642..0272bffe4b7 100644 --- a/tests/CoreFX/runtest/runtest.cmd +++ b/tests/CoreFX/runtest/runtest.cmd @@ -65,10 +65,6 @@ if not defined VS%__VSProductVersion%COMNTOOLS goto NoVS set __VSToolsRoot=!VS%__VSProductVersion%COMNTOOLS! if %__VSToolsRoot:~-1%==\ set "__VSToolsRoot=%__VSToolsRoot:~0,-1%" -set _msbuildexe="%VSINSTALLDIR%\MSBuild\15.0\Bin\MSBuild.exe" - -if not exist !_msbuildexe! (echo Error: Could not find MSBuild.exe. Please see https://github.com/dotnet/corert/blob/master/Documentation/prerequisites-for-building.md for build instructions. && exit /b 1) - :: Set the environment for the build- VS cmd prompt echo %__MsgPrefix%Using environment: "%__VSToolsRoot%\VsDevCmd.bat" call "%__VSToolsRoot%\VsDevCmd.bat" @@ -78,18 +74,6 @@ if not defined VSINSTALLDIR ( exit /b 1 ) -:: Note: We've disabled node reuse because it causes file locking issues. -:: The issue is that we extend the build with our own targets which -:: means that that rebuilding cannot successfully delete the task -:: assembly. -set __msbuildCommonArgs=/nologo /nodeReuse:false %__msbuildExtraArgs% /p:Platform=%__MSBuildBuildArch% - -if not defined __Sequential ( - set __msbuildCommonArgs=%__msbuildCommonArgs% /maxcpucount -) else ( - set __msbuildCommonArgs=%__msbuildCommonArgs% /p:ParallelRun=false -) - :: Iterate through unzipped CoreFX tests for /D %%i in ("%XunitTestBinBase%\*" ) do ( set TestFolderName=%%i @@ -98,3 +82,21 @@ for /D %%i in ("%XunitTestBinBase%\*" ) do ( echo %FXCustomTestLauncher% !TestFolderName! !TestFileName! call %FXCustomTestLauncher% !TestFolderName! !TestFileName! ) + +exit /b 0 + +:Usage +echo. +echo Usage: +echo %0 BuildArch BuildType [SkipWrapperGeneration] [Exclude EXCLUSION_TARGETS] [TestEnv TEST_ENV_SCRIPT] [VSVersion] CORE_ROOT +echo where: +echo. +echo./? -? /h -h /help -help: view this message. +echo BuildArch- Optional parameter - x64 or x86 ^(default: x64^). +echo BuildType- Optional parameter - Debug, Release, or Checked ^(default: Debug^). +exit /b 1 + +:NoVS +echo Visual Studio 2017 ^(Community is free^) is a prerequisite to build this repository. +echo See: https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/developer-guide.md#prerequisites +exit /b 1 \ No newline at end of file diff --git a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs index 589556fef7a..5bb912105d4 100644 --- a/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs +++ b/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs @@ -1,4 +1,13 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +/// +/// This helper class is used to fetch CoreFX tests from a specified URL, unarchive them and create a flat directory structure +/// through which to iterate. +/// + +using System; using System.Collections.Generic; using System.CommandLine; using System.Diagnostics; diff --git a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs index e0440f5ac8a..f7e16e4124a 100644 --- a/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs +++ b/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs @@ -1,4 +1,11 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +/// +/// This helper class reads xunit xml log files and prints the tallied test results to the console +/// +using System; using System.Collections.Generic; using System.CommandLine; using System.Diagnostics; diff --git a/tests/runtest.cmd b/tests/runtest.cmd index 322d95f55d2..8b71fefd790 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -341,15 +341,21 @@ goto :eof :: Explicitly restore the test helper project "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "%CoreRT_TestFileHelperProjectPath%" + if errorlevel 1 ( + exit /b 1 + ) "%CoreRT_CliDir%\dotnet.exe" msbuild /t:Restore "%CoreRT_XunitHelperProjectPath%" if errorlevel 1 ( exit /b 1 ) - :: Build the test helper project + :: Build the test helper projects "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_TestFileHelperProjectPath%" + if errorlevel 1 ( + exit /b 1 + ) + "%CoreRT_CliDir%\dotnet.exe" msbuild /m /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:Configuration=%CoreRT_BuildType%" "/p:OSGroup=%CoreRT_BuildOS%" "/p:Platform=%CoreRT_BuildArch%" "/p:RepoLocalBuild=true" "/p:FrameworkLibPath=%CoreRT_TestRoot%..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" "/p:OutputPath=%CoreRT_TestingUtilitiesOutputDir%" /p:RepoLocalBuild=true "%CoreRT_XunitHelperProjectPath%" - if errorlevel 1 ( exit /b 1 ) From d1cb32e68ab7658993ff66a8d5bab71ed945c79d Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Tue, 30 Jan 2018 11:31:46 -0800 Subject: [PATCH 16/16] Intermittent checkin - Build helper projects --- tests/CoreFX/corerun | 82 +++++++++++++++++++++++++++++++ tests/CoreFX/runtest/runtest.sh | 0 tests/runtest.sh | 85 +++++++++++++++++++++++++++++++-- 3 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 tests/CoreFX/corerun create mode 100644 tests/CoreFX/runtest/runtest.sh diff --git a/tests/CoreFX/corerun b/tests/CoreFX/corerun new file mode 100644 index 00000000000..dc58277fc74 --- /dev/null +++ b/tests/CoreFX/corerun @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +# This is the Unix equivalent of build-and-run-test.cmd +# It is invoked by each test's bash script. The reason it's called corerun is that +# the unix CoreCLR tests don't have a custom runner override environment variable. +# See issue https://github.com/dotnet/coreclr/issues/9007 +# The CoreFX alternative is named corerun to keep parity with CoreCLR's testing infrastructure + +#!/usr/bin/env bash + +# This is the Unix equivalent of build-and-run-test.cmd +# It is invoked by each test's bash script. The reason it's called corerun is that +# the unix CoreCLR tests don't have a custom runner override environment variable. +# See issue https://github.com/dotnet/coreclr/issues/9007 + +export TestExecutable=$1 +export TestFileName=${TestExecutable%.*} + +source "$CoreRT_TestRoot/coredump_handling.sh" + +if [[ $CoreRT_EnableCoreDumps == 1 ]]; then + set_up_core_dump_generation +fi + +cp -ar $CoreRT_TestRoot/CoreCLR/. . + +# Workaround until we have a better reflection engine +# Add name of currently executing test to rd.xml + +$__dotnetclipath/dotnet publish + + +__msbuild_dir=${CoreRT_TestRoot}/../Tools +echo ${__msbuild_dir}/msbuild.sh /m /p:IlcPath=${CoreRT_ToolchainDir} /p:Configuration=${CoreRT_BuildType} Test.csproj +${__msbuild_dir}/msbuild.sh /m /p:IlcPath=${CoreRT_ToolchainDir} /p:Configuration=${CoreRT_BuildType} Test.csproj + +if [[ $CoreRT_EnableCoreDumps == 1 ]]; then + # Handle any core files generated when running the test IL through the toolchain. + inspect_and_delete_core_files $CoreRT_ToolchainDir/corerun "$CoreRT_ToolchainDir" +fi + +# Remove the test executable from the arg list so it isn't passed to test execution +shift + +testExtRepo=$( dirname ${CoreRT_TestRoot} )/tests_downloaded/CoreCLR/ +nativeArtifactRepo=${testExtRepo}native/ +dirSuffix=$(dirname ${PWD#$testExtRepo})/ +nativeDir=${nativeArtifactRepo}tests/src/${dirSuffix} + +# In OSX we copy the native component to the directory where the exectuable resides. +# However, in Linux dlopen doesn't seem to look for current directory to resolve the dynamic library. +# So instead we point LD_LIBRARY_PATH to the directory where the native component is. +if [ -e ${nativeDir} ]; then + if [ "${CoreRT_BuildOS}" == "OSX" ]; then + echo "Copying native component from :"${nativeDir} + cp ${nativeDir}*.dylib native/ 2>/dev/null + fi + if [ "${CoreRT_BuildOS}" == "Linux" ]; then + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${nativeDir} + export LD_LIBRARY_PATH + fi +fi + +if [[ ! -f native/${TestFileName} ]]; then + echo "ERROR: Native binary not found. Unable to run test." + exit -1 +fi + +pushd native/ +./${TestFileName} "$@" +testScriptExitCode=$? +popd + +if [[ $CoreRT_EnableCoreDumps == 1 ]]; then + # Handle any core files generated when running the test. + inspect_and_delete_core_files native/$TestFileName "$CoreRT_ToolchainDir" +fi + +# Clean up test binary artifacts to save space +rm -r native 2>/dev/null + +exit $testScriptExitCode diff --git a/tests/CoreFX/runtest/runtest.sh b/tests/CoreFX/runtest/runtest.sh new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/runtest.sh b/tests/runtest.sh index 15bff4f2e6e..4d531049583 100755 --- a/tests/runtest.sh +++ b/tests/runtest.sh @@ -6,6 +6,7 @@ usage() echo " -mode : Compilation mode. Specify cpp/ryujit. Default: ryujit" echo " -test : Run a single test by folder name (ie, BasicThreading)" echo " -runtest : Should just compile or run compiled binary? Specify: true/false. Default: true." + echo " -corefx : Download and run the CoreFX repo tests" echo " -coreclr : Download and run the CoreCLR repo tests" echo " -multimodule : Compile the framework as a .so and link tests against it (ryujit only)" echo " -coredumps : [For CI use] Enables core dump generation, and analyzes and possibly stores/uploads" @@ -81,7 +82,7 @@ run_test_dir() return $? } -download_and_unzip_tests_artifacts() +download_and_unzip_coreclr_tests_artifacts() { url=$1 location=$2 @@ -101,6 +102,38 @@ download_and_unzip_tests_artifacts() fi } +download_and_unzip_corefx_tests_artifacts() +{ + # Build and restore test helper projects + ${__msbuild_dir}/dotnetcli/dotnet ${__msbuild_dir}/MSBuild.dll /t:Restore "${CoreRT_TestFileHelperProjectPath}" + __exitcode = $? + if [__exitcode != 0]; then + exit __exitcode + fi + + ${__msbuild_dir}/dotnetcli/dotnet ${__msbuild_dir}/MSBuild.dll /t:Restore "${CoreRT_XunitHelperProjectPath}" + __exitcode = $? + if [__exitcode != 0]; then + exit __exitcode + fi + + # Build the test helper projects + ${__msbuild_dir}/dotnetcli/dotnet ${__msbuild_dir}/MSBuild.dll /m /ConsoleLoggerParameters:ForceNoAlign /p:IlcPath=${CoreRT_ToolchainDir} /p:Configuration=${CoreRT_BuildType} /p:Platform=${CoreRT_BuildArch} /p:OSGroup=${CoreRT_BuildOS} /p:RepoLocalBuild=true "/p:FrameworkLibPath=${CoreRT_TestRoot}/../bin/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/lib" "/p:FrameworkObjPath=${CoreRT_TestRoot}/../bin/obj/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/Framework" "${CoreRT_TestFileHelperProjectPath}" + + __exitcode = $? + if [__exitcode != 0]; then + exit __exitcode + fi + + ${__msbuild_dir}/dotnetcli/dotnet ${__msbuild_dir}/MSBuild.dll /m /ConsoleLoggerParameters:ForceNoAlign /p:IlcPath=${CoreRT_ToolchainDir} /p:Configuration=${CoreRT_BuildType} /p:Platform=${CoreRT_BuildArch} /p:OSGroup=${CoreRT_BuildOS} /p:RepoLocalBuild=true "/p:FrameworkLibPath=${CoreRT_TestRoot}/../bin/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/lib" "/p:FrameworkObjPath=${CoreRT_TestRoot}/../bin/obj/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/Framework" "${CoreRT_XunitHelperProjectPath}" + __exitcode = $? + if [__exitcode != 0]; then + exit __exitcode + fi + +} + + restore_coreclr_tests() { CoreRT_Test_Download_Semaphore=${CoreRT_TestExtRepo}/init-tests.completed @@ -115,10 +148,10 @@ restore_coreclr_tests() CoreRT_NativeArtifactRepo=${CoreRT_TestExtRepo}/native echo "Restoring tests (this may take a few minutes).." - download_and_unzip_tests_artifacts ${TESTS_REMOTE_URL} ${CoreRT_TestExtRepo} ${CoreRT_Test_Download_Semaphore} + download_and_unzip_coreclr_tests_artifacts ${TESTS_REMOTE_URL} ${CoreRT_TestExtRepo} ${CoreRT_Test_Download_Semaphore} echo "Restoring native test artifacts..." - download_and_unzip_tests_artifacts ${NATIVE_REMOTE_URL} ${CoreRT_NativeArtifactRepo} ${CoreRT_NativeArtifact_Download_Semaphore} + download_and_unzip_coreclr_tests_artifacts ${NATIVE_REMOTE_URL} ${CoreRT_NativeArtifactRepo} ${CoreRT_NativeArtifact_Download_Semaphore} } run_coreclr_tests() @@ -156,6 +189,43 @@ run_coreclr_tests() ./runtest.sh --testRootDir=${CoreRT_TestExtRepo} --coreOverlayDir=${CoreRT_TestRoot}/CoreCLR ${CoreRT_TestSelectionArg} --logdir=$__LogDir --disableEventLogging } +run_corefx_tests() +{ + CoreRT_TestExtRepo_CoreFX=%CoreRT_TestRoot%\..\tests_downloaded\CoreFX + CoreRT_TestingUtilitiesOutputDir=%CoreRT_TestExtRepo_CoreFX%\Utilities + + # Set paths to helpers + CoreRT_TestFileHelperName=CoreFX.TestUtils.TestFileSetup + CoreRT_TestFileHelperProjectPath="%CoreRT_TestRoot%\CoreFX\runtest\src\TestUtils\TestFileSetup\%CoreRT_TestFileHelperName%.csproj" + + CoreRT_XunitHelperName=CoreFX.TestUtils.XUnit + CoreRT_XunitHelperProjectPath="%CoreRT_TestRoot%\CoreFX\runtest\src\TestUtils\XUnit\%CoreRT_XunitHelperName%.csproj" + + download_and_unzip_corefx_tests_artifacts ${CoreRT_TestFileHelperProjectPath} ${CoreRT_XunitHelperProjectPath} + __exitcode = $? + if [__exitcode != 0]; + then + exit __exitcode + fi + # FXCustomTestLauncher=%CoreRT_TestRoot%\CoreFX\build-and-run-test.cmd + # XunitTestBinBase=%CoreRT_TestExtRepo_CoreFX% + # XunitLogDir= %__LogDir%\CoreFX + # pushd %CoreRT_TestRoot%\CoreFX\runtest + + # # TODO Add single test/target test support; add exclude tests argument + + # runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% + # runtest.cmd %CoreRT_BuildArch% %CoreRT_BuildType% LogsDir %XunitLogDir% + # if errorlevel 1 + # then + # exit /b 1 + + + # "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestingUtilitiesOutputDir!\!CoreRT_XunitHelperName!.dll --logDir "%XunitLogDir%" --pattern "*.xml" + + +} + CoreRT_TestRoot="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CoreRT_CliBinDir=${CoreRT_TestRoot}/../Tools/dotnetcli CoreRT_BuildArch=x64 @@ -229,6 +299,10 @@ while [ "$1" != "" ]; do exit -1 fi ;; + -corefx) + CoreRT_RunCoreFXTests=true; + shift + ;; -multimodule) CoreRT_MultiFileConfiguration=MultiModule; ;; @@ -315,6 +389,11 @@ if [ ${CoreRT_RunCoreCLRTests} ]; then exit $? fi +if [ ${CoreRT_RunCoreFXTests} ]; then + run_corefx_tests + exit $? +fi + __CppTotalTests=0 __CppPassedTests=0 __JitTotalTests=0