Make SmokeTests use repo infrastructure (#19290)

eng/pipelines/templates/jobs/vmr-build.yml

@@ -241,6 +241,11 @@ jobs:
         call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
       displayName: Build
 
+    - ${{ if eq(parameters.runTests, 'True') }}:
+      - script: |
+          call $(sourcesPath)\build.cmd -ci -prepareMachine -test ${{ parameters.extraProperties }}
+        displayName: Run Tests
+
   - ${{ else }}:
     - ${{ if eq(parameters.buildSourceOnly, 'true') }}:
       - script: |
@@ -356,14 +361,13 @@ jobs:
         set -ex
 
         dockerVolumeArgs="-v $(sourcesPath):/vmr"
-        dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests }} -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true -e SMOKE_TESTS_RUNNING_IN_CI=true"
         poisonArg=''
 
         if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
           poisonArg='--poison'
         fi
 
-        docker run --rm $dockerVolumeArgs -w /vmr $dockerEnvArgs ${{ parameters.container }} ./build.sh --source-only $poisonArg --test $(additionalBuildArgs) /p:SmokeTestConsoleVerbosity=detailed
+        docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --source-only --test $poisonArg $(additionalBuildArgs) /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}
       displayName: Run Tests
 
   - ${{ if eq(parameters.targetOS, 'windows') }}:
@@ -383,14 +387,14 @@ jobs:
 
         cd "$(sourcesPath)"
 
-        CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.binlog"
-        CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.log"
-        CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.diff"
+        CopyWithRelativeFolders "artifacts/log/"             $targetFolder "*.binlog"
+        CopyWithRelativeFolders "artifacts/log/"             $targetFolder "*.log"
+        CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.binlog"
+        CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.diff"
+        CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "Updated*.txt"
+        CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.trx"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.binlog"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.log"
-        CopyWithRelativeFolders "test/"                      $targetFolder "*.binlog"
-        CopyWithRelativeFolders "test/"                      $targetFolder "Updated*.diff"
-        CopyWithRelativeFolders "test/"                      $targetFolder "Updated*.txt"
 
         # check if we have assets to publish
         if (Test-Path "artifacts/assets/Release/*") {
@@ -410,18 +414,20 @@ jobs:
         mkdir -p ${targetFolder}
 
         cd "$(sourcesPath)"
-        find artifacts/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
-        find artifacts/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
-        find artifacts/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/log/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/log/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "*.trx" -exec rsync -R {} -t ${targetFolder} \;
+
         if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
           find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
           find artifacts/log/binary-report/ -exec rsync -R {} -t ${targetFolder} \;
         fi
+
         find src/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find src/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
-        find test/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
-        find test/ -type f -name "Updated*.diff" -exec rsync -R {} -t ${targetFolder} \;
-        find test/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;
 
         # check if we have assets to publish
         if [ -n "$(ls -A 'artifacts/assets/Release/')" ]; then
@@ -446,11 +452,11 @@ jobs:
       continueOnError: true
       inputs:
         testRunner: vSTest
-        testResultsFiles: 'test/**/*.trx'
+        testResultsFiles: 'artifacts/TestResults/Release/*.trx'
         searchFolder: $(sourcesPath)
         mergeTestResults: true
         publishRunAttachments: true
-        testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName)
+        testRunTitle: Tests_$(Agent.JobName)
 
   - task: CopyFiles@2
     inputs:
@@ -460,6 +466,7 @@ jobs:
         assets/**
       TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
     displayName: Copy artifacts to Artifact Staging Directory
+    condition: succeededOrFailed()
 
   # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
   - ${{ if ne(parameters.buildSourceOnly, 'true') }}:
@@ -468,6 +475,7 @@ jobs:
         SourceFolder: $(sourcesPath)/artifacts/packages
         TargetFolder: $(Build.ArtifactStagingDirectory)/publishing/packages
       displayName: Copy packages to Artifact Staging Directory
+      condition: succeededOrFailed()
 
   - ${{ if or(ne(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'PullRequest')) }}:
     - publish: $(Build.ArtifactStagingDirectory)/publishing

src/SourceBuild/content/.gitignore

@@ -3,7 +3,4 @@
 /artifacts
 /prereqs/packages
 /src/nuget-client/NuGet.config
-/test/Microsoft.DotNet.SourceBuild.SmokeTests/bin
-/test/Microsoft.DotNet.SourceBuild.SmokeTests/obj
-/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults
 *.binlog

src/SourceBuild/content/Directory.Build.props

@@ -73,15 +73,18 @@
     <TargetRid Condition="'$(ShortStack)' == 'true' and '$(TargetOS)' == 'windows'">win-$(TargetArchitecture)</TargetRid>
   </PropertyGroup>
 
+  <!-- Set NuGetPackageRoot before Arcade SDK sets it. -->
+  <PropertyGroup>
+    <!-- Set RestorePackagesPath so that we don't accidentally pull some packages from the global location. -->
+    <RestorePackagesPath Condition="'$(RestorePackagesPath)' == ''">$([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', '.packages'))</RestorePackagesPath>
+    <NuGetPackageRoot Condition="'$(NuGetPackageRoot)' == ''">$(RestorePackagesPath)</NuGetPackageRoot>
+  </PropertyGroup>
+
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" Condition="'$(SkipArcadeSdkImport)' != 'true'" />
 
   <!-- Init basic Arcade props, if the project importing this file doesn't use Arcade.
        Keep in sync with props/targets in the Arcade.Sdk. -->
   <PropertyGroup Condition="'$(SkipArcadeSdkImport)' == 'true'">
-    <!-- RepoLayout.props -->
-    <NuGetPackageRoot Condition="'$(NuGetPackageRoot)' != ''">$([MSBuild]::NormalizeDirectory('$(NuGetPackageRoot)'))</NuGetPackageRoot>
-    <NuGetPackageRoot Condition="'$(NuGetPackageRoot)' == '' and '$(NUGET_PACKAGES)' != ''">$([MSBuild]::NormalizeDirectory('$(NUGET_PACKAGES)'))</NuGetPackageRoot>
-
     <RepoRoot Condition="'$(RepoRoot)' == ''">$([MSBuild]::NormalizeDirectory('$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildProjectDirectory), 'global.json'))'))</RepoRoot>
 
     <!-- Respect environment variable for the .NET install directory if set; otherwise, use the repo default location -->
@@ -212,6 +215,7 @@
     <PoisonMarkerFile>.prebuilt.xml</PoisonMarkerFile>
     <PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>
     <PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
+    <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
   </PropertyGroup>
 
 </Project>

src/SourceBuild/content/Directory.Build.targets

@@ -6,22 +6,24 @@
     <PropertyGroup>
       <SdkFilenamePrefix>dotnet-sdk-</SdkFilenamePrefix>
     </PropertyGroup>
+
     <ItemGroup>
       <SdkTarballItem Include="$(ArtifactsAssetsDir)Sdk/**/$(SdkFilenamePrefix)*$(ArchiveExtension)" />
     </ItemGroup>
 
     <!--
       Extract SDK version from SDK tarball filename.
+      Keep in sync with dotnet-sdk's archive location and filename.
 
       Example:
-      dotnet-sdk-9.0.100-alpha.1.24057.1-fedora.38-x64.tar.gz
-      dotnet-sdk-<SdkVersion>-<TargetRid><ArchiveExtension>
+      artifacts\assets\<config>\Sdk\9.0.100-alpha.1.24057.1\dotnet-sdk-9.0.100-alpha.1.24057.1-fedora.38-x64.tar.gz
+      artifacts\assets\<config>\Sdk\<SdkVersion>\dotnet-sdk-<SdkVersion>-<TargetRid><ArchiveExtension>
     -->
     <PropertyGroup>
       <SdkFilename>%(SdkTarballItem.Filename)%(SdkTarballItem.Extension)</SdkFilename>
-      <SdkTarballPath>%(SdkTarballItem.Identity)</SdkTarballPath>
+      <SdkTarballPath Condition="'$(SdkTarballPath)' == ''">%(SdkTarballItem.Identity)</SdkTarballPath>
       <SourceBuiltSdkVersion>$(SdkFilename.Replace('$(SdkFilenamePrefix)','').Replace('-$(TargetRid)$(ArchiveExtension)',''))</SourceBuiltSdkVersion>
     </PropertyGroup>
   </Target>
 
-</Project>
+</Project>

src/SourceBuild/content/Directory.Packages.props

@@ -16,6 +16,10 @@
     <PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="$(MicrosoftExtensionsFileSystemGlobbingVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" />
+    <!-- External dependencies -->
+    <PackageVersion Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
+    <PackageVersion Include="xunit.extensibility.core" Version="$(XUnitVersion)" />
+    <PackageVersion Include="xunit.extensibility.execution" Version="$(XUnitVersion)" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/build.proj

@@ -1,34 +1,24 @@
-<Project>
-  <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
+<Project Sdk="Microsoft.Build.Traversal">
 
-  <PropertyGroup>
-    <!-- Fake, to satisfy the SDK. -->
-    <TargetFramework>netstandard2.0</TargetFramework>
-    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
-  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(ToolsDir)init-build.proj" BuildInParallel="false" />
+    <ProjectReference Include="$(RepoProjectsDir)$(RootRepo).proj" BuildInParallel="false" />
+  </ItemGroup>
 
-  <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
-
-  <Target Name="Build">
+  <Target Name="PrintInfo" BeforeTargets="Build">
     <PropertyGroup>
       <BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' == 'true'">source-build</BuildModeInfoText>
       <BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' != 'true'">non-source-build</BuildModeInfoText>
     </PropertyGroup>
 
     <Message Text="Build Mode: $(BuildModeInfoText)" Importance="high" />
     <Message Text="Build Environment: $(TargetArchitecture) $(Configuration) $(TargetOS) $(TargetRid)" Importance="high" />
-
-    <MSBuild Projects="$(ToolsDir)init-build.proj;
-                       $(RepoProjectsDir)$(RootRepo).proj"
-             Targets="Build"
-             BuildInParallel="false"
-             StopOnFirstFailure="true" />
   </Target>
 
   <!-- Create a merge manifest from the individual repository manifest files. -->
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="MergeAssetManifests" AfterTargets="Build">
-    <PropertyGroup>      
+    <PropertyGroup>
       <MergedAssetManifestOutputPath>$(ArtifactsDir)VerticalManifest.xml</MergedAssetManifestOutputPath>
     </PropertyGroup>
 
@@ -48,8 +38,8 @@
   <!-- Intentionally below the import to appear at the end. -->
   <Target Name="LogBuildOutputFolders"
           AfterTargets="Build">
-   <Message Importance="high" Text="Shipping packages are located in '$(ArtifactsShippingPackagesDir)'." />
-   <Message Importance="high" Text="Shipping assets are located in '$(ArtifactsAssetsDir)'." />
+    <Message Importance="high" Text="Shipping packages are located in '$(ArtifactsShippingPackagesDir)'." />
+    <Message Importance="high" Text="Shipping assets are located in '$(ArtifactsAssetsDir)'." />
   </Target>
 
 </Project>

src/SourceBuild/content/build.sh

@@ -18,7 +18,7 @@ usage()
   echo "Actions:"
   echo "  --clean                         Clean the solution"
   echo "  --help                          Print help and exit (short: -h)"
-  echo "  --test                          Run smoke tests (short: -t)"
+  echo "  --test                          Run tests (short: -t)"
   echo ""
 
   echo "Source-only settings:"
@@ -33,7 +33,7 @@ usage()
   echo ""
 
   echo "Advanced settings:"
-  echo "  --build-tests                   Build repository tests. May not be supported with --source-only"
+  echo "  --build-repo-tests              Build repository tests. May not be supported with --source-only"
   echo "  --ci                            Set when running on CI server"
   echo "  --clean-while-building          Cleans each repo after building (reduces disk space usage, short: -cwb)"
   echo "  --excludeCIBinarylog            Don't output binary log (short: -nobl)"
@@ -56,10 +56,7 @@ while [[ -h "$source" ]]; do
 done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
-# Set the NUGET_PACKAGES dir so that we don't accidentally pull some packages from the global location,
-# They should be pulled from the local feeds.
 packagesRestoredDir="$scriptroot/.packages/"
-export NUGET_PACKAGES=$packagesRestoredDir/
 
 # Common settings
 binary_log=false
@@ -112,8 +109,6 @@ while [[ $# > 0 ]]; do
       exit 0
       ;;
     -test|-t)
-      export NUGET_PACKAGES=$NUGET_PACKAGES/smoke-tests
-      properties="$properties /t:RunSmokeTest"
       test=true
       ;;
 
@@ -180,7 +175,6 @@ while [[ $# > 0 ]]; do
     -use-mono-runtime)
       properties="$properties /p:SourceBuildUseMonoRuntime=true"
       ;;
-
     *)
       properties="$properties $1"
       ;;
@@ -195,19 +189,37 @@ if [[ "$ci" == true ]]; then
   fi
 fi
 
+# Never use the global nuget cache folder
+use_global_nuget_cache=false
+
 . "$scriptroot/eng/common/tools.sh"
 
+project="$scriptroot/build.proj"
+targets="/t:Build"
+
+# This repo uses the VSTest integration instead of the Arcade Test target
+if [[ "$test" == true ]]; then
+  project="$scriptroot/test/tests.proj"
+  targets="$targets;VSTest"
+fi
+
 function Build {
   if [[ "$sourceOnly" != "true" ]]; then
 
     InitializeToolset
 
+    # Manually unset NUGET_PACKAGES as InitializeToolset sets it unconditionally.
+    # The env var shouldn't be set so that the RestorePackagesPath msbuild property is respected.
+    unset NUGET_PACKAGES
+
     local bl=""
     if [[ "$binary_log" == true ]]; then
       bl="/bl:\"$log_dir/Build.binlog\""
     fi
 
-    MSBuild "$scriptroot/build.proj" \
+    MSBuild --restore \
+      $project \
+      $targets \
       $bl \
       /p:Configuration=$configuration \
       $properties
@@ -220,22 +232,22 @@ function Build {
       properties="$properties /p:ContinuousIntegrationBuild=true"
     fi
 
-    "$CLI_ROOT/dotnet" build-server shutdown
-
-    if [ "$test" == "true" ]; then
-      "$CLI_ROOT/dotnet" msbuild "$scriptroot/build.proj" -bl:"$scriptroot/artifacts/log/$configuration/BuildTests.binlog" -flp:"LogFile=$scriptroot/artifacts/log/$configuration/BuildTests.log" -clp:v=m $properties
-    else
+    if [ "$test" != "true" ]; then
+      "$CLI_ROOT/dotnet" build-server shutdown
       "$CLI_ROOT/dotnet" msbuild "$scriptroot/eng/tools/init-build.proj" -bl:"$scriptroot/artifacts/log/$configuration/BuildMSBuildSdkResolver.binlog" -flp:LogFile="$scriptroot/artifacts/log/$configuration/BuildMSBuildSdkResolver.log" /t:ExtractToolsetPackages,BuildMSBuildSdkResolver $properties
-
       # kill off the MSBuild server so that on future invocations we pick up our custom SDK Resolver
       "$CLI_ROOT/dotnet" build-server shutdown
+    fi
 
-      # Point MSBuild to the custom SDK resolvers folder, so it will pick up our custom SDK Resolver
-      export MSBUILDADDITIONALSDKRESOLVERSFOLDER="$scriptroot/artifacts/toolset/VSSdkResolvers/"
+    # Point MSBuild to the custom SDK resolvers folder, so it will pick up our custom SDK Resolver
+    export MSBUILDADDITIONALSDKRESOLVERSFOLDER="$scriptroot/artifacts/toolset/VSSdkResolvers/"
 
-      "$CLI_ROOT/dotnet" msbuild "$scriptroot/build.proj" -bl:"$scriptroot/artifacts/log/$configuration/Build.binlog" -flp:"LogFile=$scriptroot/artifacts/log/$configuration/Build.log" $properties
+    local bl=""
+    if [[ "$binary_log" == true ]]; then
+      bl="/bl:\"$log_dir/Build.binlog\""
     fi
 
+    "$CLI_ROOT/dotnet" msbuild --restore "$project" $bl $targets $properties
   fi
 }
 

src/SourceBuild/content/eng/Versions.props

@@ -25,13 +25,15 @@
     -->
     <PrivateSourceBuiltSdkVersion>9.0.100-preview.4.24179.1</PrivateSourceBuiltSdkVersion>
     <PrivateSourceBuiltArtifactsVersion>9.0.100-preview.4.24179.1</PrivateSourceBuiltArtifactsVersion>
-    <!-- msbuild -->
-    <MicrosoftBuildVersion>17.8.3</MicrosoftBuildVersion>
-    <!-- runtime -->
-    <MicrosoftExtensionsFileSystemGlobbingVersion>9.0.0-preview.2.24128.5</MicrosoftExtensionsFileSystemGlobbingVersion>
-    <MicrosoftExtensionsLoggingConsoleVersion>9.0.0-preview.2.24128.5</MicrosoftExtensionsLoggingConsoleVersion>
-    <MicrosoftExtensionsLoggingVersion>9.0.0-preview.2.24128.5</MicrosoftExtensionsLoggingVersion>
-    <!-- command-line-api -->
+    <!-- command-line-api dependencies -->
     <SystemCommandLineVersion>2.0.0-beta4.24126.1</SystemCommandLineVersion>
+    <!-- msbuild dependencies -->
+    <MicrosoftBuildVersion>17.8.3</MicrosoftBuildVersion>
+    <!-- runtime dependencies -->
+    <MicrosoftExtensionsFileSystemGlobbingVersion>8.0.0</MicrosoftExtensionsFileSystemGlobbingVersion>
+    <MicrosoftExtensionsLoggingConsoleVersion>8.0.0</MicrosoftExtensionsLoggingConsoleVersion>
+    <MicrosoftExtensionsLoggingVersion>8.0.0</MicrosoftExtensionsLoggingVersion>
+    <!-- external dependencies -->
+    <NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
   </PropertyGroup>
 </Project>