Enable scenario test execution in VMR build

eng/pipelines/templates/jobs/vmr-build.yml

@@ -363,7 +363,7 @@ jobs:
           set -ex
 
           dockerVolumeArgs="-v $(sourcesPath):/vmr"
-          sourceOnlyArgs=''
+          customBuildArgs=''
           extraBuildProperties=''
 
           if [[ ! -z '${{ parameters.targetOS }}' ]]; then
@@ -374,11 +374,15 @@ jobs:
             extraBuildProperties="$extraBuildProperties /p:TargetArchitecture=${{ parameters.targetArchitecture }}"
           fi
 
+          if [[ '${{ parameters.useDevVersions }}' == 'True' ]]; then
+            customBuildArgs="$customBuildArgs --dev"
+          fi
+
           if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
             if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
-              sourceOnlyArgs='--poison'
+              customBuildArgs="$customBuildArgs --poison"
             fi
-            sourceOnlyArgs="$sourceOnlyArgs --source-only /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}"
+            customBuildArgs="$customBuildArgs --source-only /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}"
           fi
 
           if [[ -n "${{ parameters.extraProperties }}" ]]; then
@@ -387,10 +391,10 @@ jobs:
 
           # Only use Docker when a container is specified
           if [[ -n "${{ parameters.container }}" ]]; then
-            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --test $sourceOnlyArgs $extraBuildProperties $(additionalBuildArgs) 
+            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs) 
           else
             cd $(sourcesPath)
-            ./build.sh /bl:artifacts/log/Release/Test.binlog --test $sourceOnlyArgs $extraBuildProperties $(additionalBuildArgs)
+            ./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
           fi
 
         displayName: Run Tests
@@ -474,18 +478,31 @@ jobs:
 
   # Only upload test results if enabled
   - ${{ if eq(parameters.runTests, 'True') }}:
+
     - task: PublishTestResults@2
       displayName: Publish Test Results
       condition: succeededOrFailed()
       continueOnError: true
       inputs:
-        testRunner: vSTest
+        testRunner: VSTest
         testResultsFiles: 'artifacts/TestResults/Release/*.trx'
         searchFolder: $(sourcesPath)
         mergeTestResults: true
         publishRunAttachments: true
         testRunTitle: Tests_$(Agent.JobName)
 
+    - task: PublishTestResults@2
+      displayName: Publish Scenario Test Results
+      condition: succeededOrFailed()
+      continueOnError: true
+      inputs:
+        testRunner: xUnit
+        testResultsFiles: 'artifacts/TestResults/**/scenario-tests/*.xml'
+        searchFolder: $(sourcesPath)
+        mergeTestResults: true
+        publishRunAttachments: true
+        testRunTitle: ScenarioTests_$(Agent.JobName)
+
   - task: CopyFiles@2
     inputs:
       SourceFolder: $(sourcesPath)/artifacts

eng/pipelines/templates/stages/vmr-build.yml

@@ -123,6 +123,7 @@ stages:
           artifactsRid: alpine.3.19-x64
           pool: ${{ parameters.pool_Linux }}
           container: ${{ variables.alpine319Container }}
+          targetOS: linux-musl
           buildFromArchive: false            # 🚫
           buildSourceOnly: true              # ✅
           enablePoison: true                 # ✅
@@ -143,6 +144,7 @@ stages:
             architecture: x64
             pool: ${{ parameters.pool_Linux }}
             container: ${{ variables.alpine319Container }}
+            targetOS: linux-musl
             buildFromArchive: false            # 🚫
             buildSourceOnly: true              # ✅
             enablePoison: false                # 🚫

src/SourceBuild/content/Directory.Build.props

@@ -160,6 +160,7 @@
     <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
     <AssetManifestsIntermediateDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'AssetManifests'))</AssetManifestsIntermediateDir>
     <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
+    <DotNetSdkExtractDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'extracted-dotnet-sdk'))</DotNetSdkExtractDir>
 
     <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
     <PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>

src/SourceBuild/content/build.sh

@@ -162,7 +162,7 @@ while [[ $# > 0 ]]; do
       ;;
 
     # Advanced settings
-    -build-tests)
+    -build-repo-tests)
       properties="$properties /p:DotNetBuildTests=true"
       ;;
     -ci)

src/SourceBuild/content/eng/build.ps1

@@ -59,7 +59,7 @@ $targets = "/t:Build"
 # This repo uses the VSTest integration instead of the Arcade Test target
 if ($test) {
   $project = Join-Path (Join-Path $RepoRoot "test") "tests.proj"
-  $targets += ";VSTest"
+  $targets = "$targets;VSTest"
 }
 
 if ($buildRepoTests) {

src/SourceBuild/content/eng/extract-sdk-archive.proj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.Build.NoTargets">
+
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+  </PropertyGroup>
+
+  <Target Name="ExtractSdkArchive" BeforeTargets="AfterBuild" DependsOnTargets="DetermineSourceBuiltSdkVersion">
+
+    <MakeDir Directories="$(DotNetSdkExtractDir)" />
+    <Exec Condition="'$(ArchiveExtension)' == '.tar.gz'"
+          Command="tar -xzf $(SdkTarballPath) -C $(DotNetSdkExtractDir)" />
+    <Unzip Condition="'$(ArchiveExtension)' == '.zip'"
+           SourceFiles="$(SdkTarballPath)"
+           DestinationFolder="$(DotNetSdkExtractDir)" />
+
+  </Target>
+
+</Project>

src/SourceBuild/content/eng/finish-source-only.proj

@@ -10,6 +10,7 @@
   <ItemGroup>
     <ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
     <ProjectReference Include="$(TasksDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj" />
+    <ProjectReference Include="$(RepositoryEngineeringDir)extract-sdk-archive.proj" />
     <ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
   </ItemGroup>
 
@@ -59,13 +60,9 @@
           Outputs="$(SdkSymbolsTarball)">
     <PropertyGroup>
       <IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
-      <IntermediateSdkLayout>$(BaseIntermediateOutputPath)Sdk</IntermediateSdkLayout>
     </PropertyGroup>
 
-    <MakeDir Directories="$(IntermediateSdkLayout)" />
-    <Exec Command="tar -xzf $(SdkTarballPath) -C $(IntermediateSdkLayout)" />
-
-    <CreateSdkSymbolsLayout SdkLayoutPath="$(IntermediateSdkLayout)"
+    <CreateSdkSymbolsLayout SdkLayoutPath="$(DotNetSdkExtractDir)"
                             AllSymbolsPath="$(IntermediateSymbolsRootDir)"
                             SdkSymbolsLayoutPath="$(IntermediateSdkSymbolsLayout)"
                             FailOnMissingPDBs="false" />
@@ -76,7 +73,6 @@
     <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
 
     <RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
-    <RemoveDir Directories="$(IntermediateSdkLayout)" />
   </Target>
 
   <!--

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/CreateSdkSymbolsLayout.cs

@@ -89,8 +89,9 @@ private IList<string> GenerateSymbolsLayout(Hashtable allPdbGuids)
                     {
                         // Copy matching pdb to symbols path, preserving sdk binary's hierarchy
                         string sourcePath = (string)allPdbGuids[debugId]!;
+                        string fileRelativePath = file.Substring(SdkLayoutPath.Length);
                         string destinationPath =
-                            file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath)
+                            Path.Combine(SdkSymbolsLayoutPath, fileRelativePath)
                                 .Replace(Path.GetFileName(file), Path.GetFileName(sourcePath));
 
                         Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -94,6 +94,17 @@
     <BuildArgs>$(BuildArgs) /p:ReferencePackageNupkgCacheDir="$(ReferencePackagesDir)"</BuildArgs>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <TestActions>$(FlagParameterPrefix)restore</TestActions>
+    <TestActions>$(TestActions) $(FlagParameterPrefix)test</TestActions>
+
+    <TestArgs>$(FlagParameterPrefix)ci</TestArgs>
+    <TestArgs>$(TestArgs) $(FlagParameterPrefix)configuration $(Configuration)</TestArgs>
+    <TestArgs>$(TestArgs) -bl:artifacts/log/$(Configuration)/Test.binlog</TestArgs>
+
+    <TestCommand>$(BuildScript) $(TestActions) $(TestArgs)</TestCommand>
+  </PropertyGroup>
+
   <PropertyGroup Condition="'$(EnableExtraDebugging)' == 'true'">
     <MSBuildDebugPathTargetDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'msbuild-debug'))</MSBuildDebugPathTargetDir>
     <RoslynDebugPathTargetDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'roslyn-debug'))</RoslynDebugPathTargetDir>

src/SourceBuild/content/repo-projects/installer.proj

@@ -79,10 +79,12 @@
   <Target Name="CopySdkArchive"
           AfterTargets="Build"
           DependsOnTargets="DetermineSourceBuiltSdkVersion">
+
     <Copy SourceFiles="$(SdkTarballPath)"
           DestinationFolder="$(ArtifactsAssetsDir)"
           SkipUnchangedFiles="true"
           UseHardlinksIfPossible="true" />
+
   </Target>
 
 </Project>

src/SourceBuild/content/repo-projects/scenario-tests.proj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.Build.NoTargets">
+<Project Sdk="Microsoft.Build.NoTargets" TreatAsLocalProperty="CleanWhileBuilding">
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
@@ -10,4 +10,58 @@
     <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <!-- The scenario-tests repo shouldn't be cleaned after building as we run tests from it. -->
+    <CleanWhileBuilding>false</CleanWhileBuilding>
+
+    <ScenarioTestsArtifactsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'scenario-tests'))</ScenarioTestsArtifactsDir>
+    <ScenarioTestsResultsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsTestResultsDir)', 'scenario-tests'))</ScenarioTestsResultsDir>
+    <_ScenarioTestsNuGetConfig>$(ScenarioTestsArtifactsDir)NuGet.config</_ScenarioTestsNuGetConfig>
+    <_InstallerNuGetConfig>$(RepoRoot)src/installer/NuGet.config</_InstallerNuGetConfig>
+  </PropertyGroup>
+
+  <Target Name="SetupNuGetConfig"
+          Inputs="$(_InstallerNuGetConfig)"
+          Outputs="$(_ScenarioTestsNuGetConfig)">
+
+    <Copy SourceFiles="$(_InstallerNuGetConfig)"
+          DestinationFiles="$(_ScenarioTestsNuGetConfig)" />
+
+  </Target>
+
+  <Target Name="RunScenarioTests"
+          DependsOnTargets="SetupNuGetConfig;DetermineSourceBuiltSdkVersion">
+
+    <PropertyGroup>
+      <_CurrentDateTime>$([System.DateTime]::Now.ToString("yyyy-MM-dd_HH_mm_ss"))</_CurrentDateTime>
+      <_TestXmlOutputPath>$(ScenarioTestsResultsDir)$(_CurrentDateTime).xml</_TestXmlOutputPath>
+      <_ScenarioTestsAdditionalArgs>--xml $(_TestXmlOutputPath) --target-rid $(TargetRid) --no-cleanup --no-traits Category=MultiTFM</_ScenarioTestsAdditionalArgs>
+
+      <!-- Define the test root as a sub-directory of the scenario test artifacts directory. It needs to be a sub-directory because the scenario test execution
+           will clean that directory. Since we need the NuGet.config file that we copied in to be preserved, that's stored in the directory above the test root. -->
+      <_TestRoot>$(ScenarioTestsArtifactsDir)artifacts/</_TestRoot>
+
+      <!-- It's necessary to explicitly define the path to the dotnet tool to prevent Arcade from attempting to derive it. Otherwise, it will run the dotnet
+           install script to get a new one. We must use the locally built SDK instead in order to support non-portable RIDs for source build. -->
+      <_DotNetTool Condition="'$(BuildOS)' == 'windows'">$(DotNetSdkExtractDir)dotnet.exe</_DotNetTool>
+      <_DotNetTool Condition="'$(BuildOS)' != 'windows'">$(DotNetSdkExtractDir)dotnet</_DotNetTool>
+    </PropertyGroup>
+
+    <MakeDir Directories="$(ScenarioTestsResultsDir)" />
+
+    <ItemGroup>
+      <_ScenarioTestEnvVars Include="
+        TestRoot=$(_TestRoot);
+        DotNetRoot=$(DotNetSdkExtractDir);
+        TestSdkVersion=$(SourceBuiltSdkVersion);
+        AdditionalTestArgs=$(_ScenarioTestsAdditionalArgs);
+        DotNetTool=$(_DotNetTool);
+        _InitializeDotNetCli=$(DotNetSdkExtractDir)" />
+    </ItemGroup>
+
+    <Exec Command="$(TestCommand)"
+          WorkingDirectory="$(ProjectDirectory)"
+          EnvironmentVariables="@(_ScenarioTestEnvVars)" />
+  </Target>
+
 </Project>

src/SourceBuild/content/test/scenario-tests.proj

@@ -0,0 +1,53 @@
+<Project Sdk="Microsoft.Build.Traversal">
+
+  <PropertyGroup>
+    <ScenarioTestsArtifactsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'scenario-tests'))</ScenarioTestsArtifactsDir>
+    <ScenarioTestsResultsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsTestResultsDir)', 'scenario-tests'))</ScenarioTestsResultsDir>
+    <_ScenarioTestsNuGetConfig>$(ScenarioTestsArtifactsDir)NuGet.config</_ScenarioTestsNuGetConfig>
+    <_InstallerNuGetConfig>$(RepoRoot)src/installer/NuGet.config</_InstallerNuGetConfig>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../src/scenario-tests/src/**/*.csproj" />
+  </ItemGroup>
+
+  <Target Name="SetupNuGetConfig"
+          Inputs="$(_InstallerNuGetConfig)"
+          Outputs="$(_ScenarioTestsNuGetConfig)">
+
+    <Copy SourceFiles="$(_InstallerNuGetConfig)"
+          DestinationFiles="$(_ScenarioTestsNuGetConfig)" />
+
+  </Target>
+
+  <Target Name="RunScenarioTests"
+          DependsOnTargets="SetupNuGetConfig;DetermineSourceBuiltSdkVersion">
+
+    <PropertyGroup>
+      <_CurrentDateTime>$([System.DateTime]::Now.ToString("yyyy-MM-dd_HH_mm_ss"))</_CurrentDateTime>
+      <_TestXmlOutputPath>$(ScenarioTestsResultsDir)$(_CurrentDateTime).xml</_TestXmlOutputPath>
+      <_ScenarioTestsAdditionalArgs>--xml $(_TestXmlOutputPath) --target-rid $(TargetRid) --no-cleanup --no-traits Category=MultiTFM</_ScenarioTestsAdditionalArgs>
+
+      <!-- Define the test root as a sub-directory of the scenario test artifacts directory. It needs to be a sub-directory because the scenario test execution
+           will clean that directory. Since we need the NuGet.config file that we copied in to be preserved, that's stored in the directory above the test root. -->
+      <_TestRoot>$(ScenarioTestsArtifactsDir)artifacts/</_TestRoot>
+    </PropertyGroup>
+
+    <MakeDir Directories="$(ScenarioTestsResultsDir)" />
+
+    <ItemGroup>
+      <_ScenarioTestProperties Include="
+        Configuration=$(Configuration);
+        RunScenarioTests=true;
+        TestRoot=$(_TestRoot);
+        DotNetRoot=$(DotNetSdkExtractDir);
+        NETCoreSdkVersion=$(SourceBuiltSdkVersion);
+        AdditionalTestArgs=$(_ScenarioTestsAdditionalArgs);
+        DOTNET_INSTALL_DIR=$(DotNetRoot)" />
+    </ItemGroup>
+
+    <MSBuild Projects="$(MSBuildThisFileFullPath)"
+             Properties="@(_ScenarioTestProperties)"
+             Targets="Restore;Build;Test" />
+  </Target>
+</Project>

src/SourceBuild/content/test/scenario-tests/Directory.Build.targets

@@ -0,0 +1,9 @@
+<Project>
+
+  <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.targets, $(MSBuildThisFileDirectory)..))" />
+
+  <!-- scenario-tests test execution requires a custom test target. -->
+  <Target Name="Test" DependsOnTargets="RunScenarioTests" />
+  <Target Name="VSTest" DependsOnTargets="RunScenarioTests" />
+
+</Project>

src/SourceBuild/content/test/scenario-tests/scenario-tests.proj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.Build.NoTargets">
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <!-- Scenarios tests are handled separately from the other test projects. This is because the output of the scenario tests is
+       an executable that is used to run the tests and so using the VSTest runner doesn't work with them. They also require
+       some setup steps to be configured with the VMR. -->
+  <Target Name="RunScenarioTests">
+    <MSBuild Projects="$(RepoProjectsDir)scenario-tests.proj"
+             Properties="Configuration=$(Configuration)"
+             Targets="RunScenarioTests" />
+  </Target>
+
+</Project>

src/SourceBuild/content/test/tests.proj

@@ -1,10 +1,29 @@
 <Project Sdk="Microsoft.Build.Traversal">
 
+  <PropertyGroup>
+    <_RunScenarioTests>true</_RunScenarioTests>
+
+    <!-- Skip scenario tests if the host architecture is different from the target architecture since the tests
+        require the ability to execute the built SDK. But the CLI is not capable of running on a host with a 
+        different architecture (i.e. "cannot execute binary file: Exec format error"). -->
+    <_RunScenarioTests Condition="'$(BuildArchitecture.ToLower())' != '$(TargetArchitecture.ToLower())'">false</_RunScenarioTests>
+
+    <!-- Skip scenario tests if the portable OS (determined from the host machine) is different from the target OS
+        since the tests require the ability to execute the built SDK. An example of where this would be disabled is
+        cross-build of using Mariner to build for Alpine (linux vs linux-musl). -->
+    <_RunScenarioTests Condition="'$(BuildOS)' != 'windows' and '$(__PortableTargetOS.ToLower())' != '$(TargetOS.ToLower())'">false</_RunScenarioTests>
+
+    <!-- The scenario tests are not supported when unofficial build versioning is used. -->
+    <_RunScenarioTests Condition="'$(UseOfficialBuildVersioning)' == 'false'">false</_RunScenarioTests>
+  </PropertyGroup>
+
   <ItemGroup>
+    <ProjectReference Include="$(RepositoryEngineeringDir)extract-sdk-archive.proj" />
     <ProjectReference Include="Microsoft.DotNet.SourceBuild.SmokeTests\Microsoft.DotNet.SourceBuild.SmokeTests.csproj"
                       Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
     <ProjectReference Include="Microsoft.DotNet.UnifiedBuild.Tests\Microsoft.DotNet.UnifiedBuild.Tests.csproj"
                       Condition="'$(ShortStack)' != 'true' and '$(PortableBuild)' == 'true' and '$(PgoInstrument)' != 'true'" />
+    <ProjectReference Include="scenario-tests\scenario-tests.proj" Condition="'$(_RunScenarioTests)' == 'true'" />
   </ItemGroup>
 
-</Project>
+</Project>