Enable CoreFX tests on CoreRT (#5329)

* Generate Test projects - Intermittent checkin * Intermittent check-in running exe, tests need restoring * Add Test utitilites project * End-to-End test build Windows * Change structure * Fix RDXML and add reflectable types * Add json parsing and xml logs * End-to-end test run and display * Helper project style changes * Remove direct dll references add package dependencies * Stylistic changes + deps * Style fixup * XML reader escape * Dependency and comment cleanup * Respond to PR feedback * Intermittent checkin - Build helper projects Shell scripts * Building and working shell scripts * End-to-end build and run * Add error checking * Style Fixup * Address PR comments * Revert line change * Remove Net.Security.Tests * OSX Changes * Update URL; update dependencies * Remove Collections.Immutable and update to long-lasting URL
dotnet · Mar 14, 2018 · bf6b057 · bf6b057
1 parent 1d5f330
commit bf6b057
Show file tree

Hide file tree

Showing 16 changed files with 1,045 additions and 14 deletions.
diff --git a/tests/CoreFX/build-and-run-test.cmd b/tests/CoreFX/build-and-run-test.cmd
@@ -0,0 +1,54 @@
+:: Test build and execution wrapper for CoreFX tests
+::
+:: This wrapper is called for each of CoreFX's tests by runtest.cmd
+::
+:: %1 contains test folder
+:: %2 contains test exe name
+::
+@echo OFF
+setlocal ENABLEDELAYEDEXPANSION
+
+set TestFolder=%1
+
+::
+:: We're only interested in referencing the xunit runner - the test dlls will be imported by the test wrapper project
+::
+set TestExecutable=xunit.console.netcore
+set TestFileName=%2
+
+
+:: Copy the artefacts we need to compile and run the xunit exe
+copy /Y "%~dp0\runtest\CoreFXTestHarness\*" "%TestFolder%" >nul
+
+:: Create log dir if it doesn't exist
+if not exist %XunitLogDir% md %XunitLogDir%
+
+if not exist %TestFolder%\%TestExecutable%.exe ( 
+    :: Not a test we support, exit silently
+    exit /b 0
+)
+
+:: Workaround until we have a better reflection engine
+:: Add name of currently executing test to rd.xml
+powershell -Command "(Get-Content %TestFolder%\default.rd.xml).replace('*Application*', '%TestFileName%') | Set-Content  %TestFolder%\default.rd.xml"
+
+if "%CoreRT_BuildArch%" == "x64" (
+    call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" >nul
+)
+
+echo Building %TestFileName%
+
+call "%CoreRT_CliDir%\dotnet.exe" publish %TestFolder%\Test.csproj /ConsoleLoggerParameters:ForceNoAlign "/p:IlcPath=%CoreRT_ToolchainDir%" "/p:DebugSymbols=false" "/p:Configuration=%CoreRT_BuildType%" "/p:FrameworkLibPath=%~dp0..\..\bin\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\lib" "/p:FrameworkObjPath=%~dp0..\..\bin\obj\%CoreRT_BuildOS%.%CoreRT_BuildArch%.%CoreRT_BuildType%\Framework" /p:DisableFrameworkLibGeneration=true /p:TestRootDir=%~dp0  /p:OSGroup=%CoreRT_BuildOS% /p:ExecutableName=%TestExecutable% /nologo
+if errorlevel 1 (
+    echo Building %TestFileName% failed
+    exit /b 1
+)
+
+echo Executing %TestFileName% - writing logs to %XunitLogDir%\%TestFileName%.xml
+
+if not exist "%TestFolder%\native\%TestExecutable%".exe (
+    echo ERROR:Native binary not found Unable to run test.
+    exit /b 1
+)
+
+call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -xml %XunitLogDir%\%TestFileName%.xml 
diff --git a/tests/CoreFX/corerun b/tests/CoreFX/corerun
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash 
+
+# This is the Unix equivalent of build-and-run-test.cmd
+# It is invoked by each test's bash script. The reason it's called corerun is that
+# the unix CoreCLR tests don't have a custom runner override environment variable.
+# See issue https://github.com/dotnet/coreclr/issues/9007
+# The CoreFX alternative is named corerun to keep parity with the CoreCLR testing convention
+
+
+export TestFolderName=$1
+export TestFileName=$(basename ${TestFolderName})
+export TestRootDir=${CoreRT_TestRoot}/CoreFX
+export LogDir=$2
+#
+# We're only interested in referencing the xunit runner - the test dlls will be imported by the test wrapper project
+#
+TestExecutable=xunit.console.netcore
+
+source "$CoreRT_TestRoot/coredump_handling.sh"
+
+if [[ $CoreRT_EnableCoreDumps == 1 ]]; then
+    set_up_core_dump_generation
+fi
+
+if [ ! -e "${TestFolderName}/${TestExecutable}.exe" ]; then 
+    echo "not found in ${TestFolderName}/${TestExecutable}.exe"
+    # Not a test we support, exit silently
+    exit 0
+fi
+
+# Copy the artefacts we need to compile and run the xunit exe
+cp -a "${CoreRT_TestRoot}/CoreFX/runtest/CoreFXTestHarness/." "${TestFolderName}"
+export __exitcode=$?
+if [ ${__exitcode} != 0 ];
+then
+    exit ${__exitcode}
+fi
+
+# Workaround until we have a better reflection engine
+# Add name of currently executing test to rd.xml
+sed -i.bak  "s/\*Application\*/${TestFileName}/g" "${TestFolderName}/default.rd.xml"
+
+echo Building ${TestFileName}
+
+${CoreRT_CliBinDir}/dotnet publish "/p:IlcPath=${CoreRT_ToolchainDir}" "/p:DebugSymbols=false" /p:OSGroup=${CoreRT_BuildOS} "/p:Configuration=${CoreRT_BuildType}" "/p:FrameworkLibPath=${CoreRT_TestRoot}/../bin/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/lib" "/p:FrameworkObjPath=${CoreRT_TestRoot}/../bin/obj/${CoreRT_BuildOS}.${CoreRT_BuildArch}.${CoreRT_BuildType}/Framework" /p:DisableFrameworkLibGeneration=true /p:TestRootDir=${TestRootDir} /p:ExecutableName=${TestExecutable} "/p:OutputPath=/${TestFolderName}/" ${TestFolderName}/Test.csproj
+export __exitcode=$?
+if [ ${__exitcode} != 0 ];
+then
+    exit ${__exitcode}
+fi
+
+echo Executing ${TestFileName} - writing logs to ${LogDir}/${TestFileName}.xml
+chmod +x ${TestFolderName}/publish/${TestExecutable} 
+${TestFolderName}/publish/${TestExecutable} ${TestFolderName}/${TestFileName}.dll -xml ${LogDir}/${TestFileName}.xml
+export __exitcode=$?
+
+exit ${__exitcode}
diff --git a/tests/CoreFX/dependencies.props b/tests/CoreFX/dependencies.props
@@ -0,0 +1,17 @@
+<Project DefaultTargets="Build" 
+    xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+    <Import Project="..\..\dependencies.props" />
+
+    <!-- Package version dependencies in test helper projects. -->
+    <PropertyGroup>
+        <SystemCommandLineVersion>0.1.0-e160909-1</SystemCommandLineVersion>
+        <NewtonsoftJsonVersion>10.0.1</NewtonsoftJsonVersion>
+        <XunitPackageVersion>2.3.0-beta1-build3642</XunitPackageVersion>     
+        <!-- Due to an API surface mismatch, if the xunit.netcore executable attempts to run the wrong version of the
+        runner utility, running the tests wil break, so separate both int odifferent version definitions  -->   
+        <XunitRunnerUtilityVersion>2.2.0-beta2-build3300</XunitRunnerUtilityVersion>                
+        <XunitAbstractionsVersion>2.0.1</XunitAbstractionsVersion>
+        <XunitNetcoreExtensionsVersion>2.1.0-preview2-02516-02</XunitNetcoreExtensionsVersion>
+        <CoreFxTestUtilitiesVersion>4.5.0-preview2-26219-0</CoreFxTestUtilitiesVersion>
+    </PropertyGroup>
+</Project>
diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj b/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj
@@ -0,0 +1,73 @@
+<Project DefaultTargets="LinkNative">
+    <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
+
+    <Import Project="$(TestRootDir)\dependencies.props" />
+
+    <!-- Override Default MSBuild properties-->
+    <PropertyGroup>
+        <TargetName>$(ExecutableName)</TargetName>
+        <TargetExt>.exe</TargetExt>
+        <OutputType>Exe</OutputType>
+    </PropertyGroup>
+
+    <PropertyGroup>
+        <ToolsDir>$(FrameworkLibPath)\..\tools\</ToolsDir>
+        <TargetFramework>netcoreapp2.0</TargetFramework>
+        <!-- Don't warn if some dependencies were rolled forward -->
+        <NoWarn>$(NoWarn);NU1603</NoWarn>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <RdXmlFile Include="default.rd.xml" />
+    </ItemGroup>
+    <ItemGroup>
+        <PackageReference Include="xunit.abstractions">
+            <Version>$(XunitAbstractionsVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.assert">
+            <Version>$(XunitPackageVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.extensibility.core">
+            <Version>$(XunitPackageVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.core">
+            <Version>$(XunitPackageVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.runner.utility">
+            <Version>$(XunitRunnerUtilityVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.extensibility.execution">
+            <Version>$(XunitPackageVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="microsoft.xunit.netcore.extensions">
+            <Version>$(XunitNetcoreExtensionsVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="CoreFx.Private.TestUtilities">
+            <Version>$(CoreFxTestUtilitiesVersion)</Version>
+        </PackageReference>
+    </ItemGroup>
+    <ItemGroup>
+        <!-- Some tests consist of multiple assemblies - make sure ILC sees them -->
+        <IlcCompileInput Include="$(MSBuildProjectDirectory)\*.dll" />
+    </ItemGroup>
+    <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
+
+    <!-- Set OutputPath after the SDK targets have been imported-->
+    <PropertyGroup>
+        <OutputPath>$(MSBuildProjectDirectory)\</OutputPath>
+        <IntermediateOutputPath>$(MSBuildProjectDirectory)\</IntermediateOutputPath>
+    </PropertyGroup>
+
+    <!-- Switch RuntimeIdentifier according to currently running OSGroup -->
+    <PropertyGroup>
+        <RuntimeIdentifier Condition="'$(OSGroup)' == 'Windows_NT'">win-x64</RuntimeIdentifier>
+        <RuntimeIdentifier Condition="'$(OSGroup)' == 'Linux'">linux-x64</RuntimeIdentifier>
+        <RuntimeIdentifier Condition="'$(OSGroup)' == 'OSX'">osx-x64</RuntimeIdentifier>
+    </PropertyGroup>
+
+    <!-- Import CoreRT build targets -->
+    <Import Project="$(IlcPath)\build\Microsoft.NETCore.Native.targets" />
+
+    <!-- Since tests are already compiled, override Compile target to prevent CSC running -->
+    <Target Name="Compile" />
+</Project>
diff --git a/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml b/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml
@@ -0,0 +1,41 @@
+<Directives 
+    xmlns="http://schemas.microsoft.com/netfx/2013/01/metadata">
+    <Application>
+        <Assembly Name="*Application*" Dynamic="Required All" />
+        <Assembly Name="xunit.abstractions" Dynamic="Required All" />
+        <Assembly Name="System.Private.CoreLib">
+            <Type Name="System.Runtime.CompilerServices.CompilationRelaxationsAttribute" Dynamic="Required All" />
+            <Type Name="System.Runtime.CompilerServices.RuntimeCompatibilityAttribute" Dynamic="Required All" />
+            <Type Name="System.Runtime.CompilerServices.ExtensionAttribute" Dynamic="Required All" />
+            <Type Name="System.Runtime.CompilerServices.IntrinsicAttribute" Dynamic="Required All" />
+            <Type Name="System.Diagnostics.DebuggableAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyInformationalVersionAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyTitleAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyFileVersionAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyDescriptionAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyCompanyAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyCopyrightAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyProductAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyDefaultAliasAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyMetadataAttribute" Dynamic="Required All" />
+            <Type Name="System.AttributeUsageAttribute" Dynamic="Required All" />
+            <Type Name="System.AttributeUsageAttribute[]" Dynamic="Required All" />
+            <Type Name="System.Runtime.InteropServices.ComVisibleAttribute" Dynamic="Required All" />
+            <Type Name="System.CLSCompliantAttribute" Dynamic="Required All" />
+        </Assembly>
+        <Assembly Name="mscorlib">
+            <Type Name="System.IO.File" Dynamic="Required All"/>
+            <Type Name="System.Threading.ExecutionContext" Dynamic="Required All"/>
+            <Type Name="System.Threading.ContextCallback" Dynamic="Required All"/>
+        </Assembly>
+        <Assembly Name="xunit.core" Dynamic="Required All" />
+        <Assembly Name="xunit.runner.utility.dotnet" Dynamic="Required All" />
+        <Assembly Name="xunit.execution.dotnet" Dynamic="Required All" />
+        <Assembly Name="xunit.abstractions" Dynamic="Required All" />
+        <Assembly Name="xunit.core" Dynamic="Required All">
+            <Type Name="Xunit.Sdk.BeforeAfterTestAttribute[]" Dynamic="Required All" />
+        </Assembly>
+        <Assembly Name="Xunit.NetCore.Extensions" Dynamic="Required All" />
+        <Assembly Name="System.Linq.Expressions" Dynamic="Required All" />
+    </Application>
+</Directives>
diff --git a/tests/CoreFX/runtest/dir.props b/tests/CoreFX/runtest/dir.props
@@ -0,0 +1,33 @@
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+
+  <!-- Don't import the project's parent dir.props - all other projects in the CoreRT repo are set up for the old project format -->
+
+  <!--
+    $(OS) is set to Unix/Windows_NT. This comes from an environment variable on Windows and MSBuild on Unix.
+  -->
+  <PropertyGroup>
+    <OsEnvironment Condition="'$(OsEnvironment)'==''">$(OS)</OsEnvironment>
+  </PropertyGroup>
+
+  <!-- Common repo directories -->
+  <PropertyGroup>
+    <CopyNuGetImplementations Condition="'$(CopyNuGetImplementations)'==''">false</CopyNuGetImplementations>
+    <ProjectDir>$(MSBuildThisFileDirectory)\</ProjectDir>
+    <SourceDir>$(ProjectDir)src\</SourceDir>
+    <PackagesDir>$(ProjectDir)..\..\..\packages\</PackagesDir>
+    <ToolsDir Condition="'$(ToolsDir)'==''">$(ProjectDir)..\Tools\</ToolsDir>
+    <DotnetCliPath Condition="'$(DotnetCliPath)'==''">$(ToolsDir)dotnetcli/</DotnetCliPath>
+    <SkipImportILTargets>true</SkipImportILTargets>
+  </PropertyGroup>
+
+  <!-- Provides properties for dependency versions and configures dependency verification/auto-upgrade. -->
+  <Import Project="$(MSBuildThisFileDirectory)\..\dependencies.props" />
+
+  <!-- list of nuget package sources passed to dnu --> 
+  <ItemGroup> 
+    <!-- Need to escape double forward slash (%2F) or MSBuild will normalize to one slash on Unix. -->
+    <DotnetSourceList Include="https:%2F%2Fdotnet.myget.org/F/dotnet-core/api/v3/index.json" />
+    <DotnetSourceList Include="https:%2F%2Fapi.nuget.org/v3/index.json" /> 
+  </ItemGroup> 
+
+</Project>
diff --git a/tests/CoreFX/runtest/runtest.cmd b/tests/CoreFX/runtest/runtest.cmd
@@ -0,0 +1,100 @@
+@if not defined __echo @echo off
+setlocal EnableDelayedExpansion
+
+:: Set the default arguments
+set __BuildArch=x64
+set __BuildType=Debug
+set __BuildOS=Windows_NT
+
+:: Default to highest Visual Studio version available
+set __VSVersion=vs2017
+set __VSProductVersion=150
+
+:: Define a prefix for most output progress messages that come from this script. That makes
+:: it easier to see where these are coming from. Note that there is a trailing space here.
+set __MsgPrefix=RUNTEST: 
+
+set __ProjectDir=%~dp0
+:: remove trailing slash
+if %__ProjectDir:~-1%==\ set "__ProjectDir=%__ProjectDir:~0,-1%"
+set "__ProjectFilesDir=%__ProjectDir%"
+set "__RootBinDir=%__ProjectDir%\..\bin"
+set "__LogsDir=%__RootBinDir%\Logs"
+
+:Arg_Loop
+if "%1" == "" goto ArgsDone
+
+if /i "%1" == "/?"    goto Usage
+if /i "%1" == "-?"    goto Usage
+if /i "%1" == "/h"    goto Usage
+if /i "%1" == "-h"    goto Usage
+if /i "%1" == "/help" goto Usage
+if /i "%1" == "-help" goto Usage
+
+if /i "%1" == "x64"                   (set __BuildArch=x64&set __MSBuildBuildArch=x64&shift&goto Arg_Loop)
+if /i "%1" == "x86"                   (set __BuildArch=x86&set __MSBuildBuildArch=x86&shift&goto Arg_Loop)
+
+if /i "%1" == "debug"                 (set __BuildType=Debug&shift&goto Arg_Loop)
+if /i "%1" == "release"               (set __BuildType=Release&shift&goto Arg_Loop)
+
+if /i "%1" == "LogsDir"         (set __LogsDir=%2&shift&shift&goto Arg_Loop) 
+
+if /i not "%1" == "msbuildargs" goto SkipMsbuildArgs
+
+set CORE_ROOT=%1
+echo %__MsgPrefix%CORE_ROOT is initially set to: "%CORE_ROOT%"
+shift 
+:ArgsDone
+
+set "__TestWorkingDir=%CoreRT_TestRoot\CoreFX%"
+
+if not defined XunitTestBinBase set  XunitTestBinBase=%__TestWorkingDir%
+
+if not exist %__LogsDir% md %__LogsDir%
+
+:: Check presence of VS
+if not defined VS%__VSProductVersion%COMNTOOLS goto NoVS
+
+set __VSToolsRoot=!VS%__VSProductVersion%COMNTOOLS!
+if %__VSToolsRoot:~-1%==\ set "__VSToolsRoot=%__VSToolsRoot:~0,-1%"
+
+:: Set the environment for the  build- VS cmd prompt
+echo %__MsgPrefix%Using environment: "%__VSToolsRoot%\VsDevCmd.bat"
+call                                 "%__VSToolsRoot%\VsDevCmd.bat"
+
+if not defined VSINSTALLDIR (
+    echo %__MsgPrefix%Error: runtest.cmd should be run from a Visual Studio Command Prompt.  Please see https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/developer-guide.md for build instructions.
+    exit /b 1
+)
+
+if not defined FXCustomTestLauncher (
+    echo The CoreFX test launcher is not defined 
+    exit /b 1
+)
+
+:: Iterate through unzipped CoreFX tests 
+for /D %%i in ("%XunitTestBinBase%\*" ) do (
+    set TestFolderName=%%i
+    set TestFileName=%%~nxi
+
+    echo %FXCustomTestLauncher% !TestFolderName! !TestFileName!
+    call %FXCustomTestLauncher% !TestFolderName! !TestFileName!
+)
+
+exit /b 0
+
+:Usage
+echo.
+echo Usage:
+echo   %0 BuildArch BuildType
+echo where:
+echo.
+echo./? -? /h -h /help -help: view this message.
+echo BuildArch- Optional parameter - x64 or x86 ^(default: x64^).
+echo BuildType- Optional parameter - Debug, Release, or Checked ^(default: Debug^).
+exit /b 1
+
+:NoVS
+echo Visual Studio 2017 ^(Community is free^) is a prerequisite to build this repository.
+echo See: https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/developer-guide.md#prerequisites
+exit /b 1