Move MSBuild projects out of the native build scripts

Directory.Build.targets

@@ -9,6 +9,7 @@
   </PropertyGroup>
 
   <Import Project="$(RepositoryEngineeringDir)/liveBuilds.targets" />
+  <Import Project="$(RepositoryEngineeringDir)/python.targets" />
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" Condition="'$(SkipImportArcadeSdkFromRoot)' != 'true'" />
 
   <PropertyGroup>

docs/workflow/building/coreclr/linux-instructions.md

@@ -124,10 +124,10 @@ To ensure that your system can allocate enough file-handles for the libraries bu
 Build the Runtime and System.Private.CoreLib
 =============================================
 
-To build the runtime on Linux, run build.sh from the root of the runtime repository:
+To build the runtime on Linux, run build.sh to build the CoreCLR subset category of the runtime:
 
 ```
-./src/coreclr/build.sh
+./build.sh -subsetCategory coreclr
 ```
 
 After the build is completed, there should some files placed in `runtime/artifacts/bin/coreclr/Linux.x64.Debug`.  The ones we are most interested in are:

docs/workflow/building/coreclr/osx-instructions.md

@@ -40,13 +40,13 @@ brew install icu4c
 brew link --force icu4c
 ```
 
-Build the Runtime and Microsoft Core Library
+Build the Runtime and System.Private.CoreLib
 ============================================
 
-To Build CoreCLR, run build.sh from the root of the coreclr repo.
+To Build CoreCLR, run build.sh to build the CoreCLR subset category of the runtime:
 
-```sh
-./src/coreclr/build.sh
+```
+./build.sh -subsetCategory coreclr
 ```
 
 After the build has completed, there should some files placed in `artifacts/bin/coreclr/OSX.x64.Debug`. The ones we are interested in are:
@@ -75,4 +75,4 @@ an environment variable to the Core_Root folder.
 ```sh
 export CORE_ROOT=/path/to/runtime/artifacts/tests/coreclr/OSX.x64.Debug/Tests/Core_Root
 $CORE_ROOT/corerun hello_world.dll
-```
+```

eng/Subsets.props

@@ -61,7 +61,7 @@
     <DefaultInstallerSubsets>corehost-managed-depproj-pkgproj-bundle-installers-test</DefaultInstallerSubsets>
     <!-- TODO: Split into multiple sets. -->
     <DefaultLibrariesSubsets>all</DefaultLibrariesSubsets>
-    <DefaultCoreClrSubsets>all</DefaultCoreClrSubsets>
+    <DefaultCoreClrSubsets>runtime-corelib-nativecorelib-tools-packages</DefaultCoreClrSubsets>
     <DefaultMonoSubsets>all</DefaultMonoSubsets>
 
     <_subsetCategory Condition="'$(SubsetCategory)' != ''">$(SubsetCategory.ToLowerInvariant())</_subsetCategory>
@@ -71,6 +71,9 @@
     <_subset Condition="'$(Subset)' == '' and $(_subsetCategory.Contains('libraries'))">$(_subset)$(DefaultLibrariesSubsets)</_subset>
     <_subset Condition="'$(Subset)' == '' and $(_subsetCategory.Contains('coreclr'))">$(_subset)$(DefaultCoreClrSubsets)</_subset>
     <_subset Condition="'$(Subset)' == '' and $(_subsetCategory.Contains('mono'))">$(_subset)$(DefaultMonoSubsets)</_subset>
+
+    <!-- Surround _subset in dashes to simplify checks below -->
+    <_subset>-$(_subset)-</_subset>
   </PropertyGroup>
 
   <ItemGroup>
@@ -81,7 +84,11 @@
     <SubsetName Include="All" Category="Libraries" Description="The .NET libraries comprising the shared framework." />
 
     <!-- CoreClr -->
-    <SubsetName Include="All" Category="CoreClr" Description="The .NET runtime." />
+    <SubsetName Include="Runtime" Category="CoreClr" Description="The CoreCLR .NET runtime." />
+    <SubsetName Include="CoreLib" Category="CoreClr" Description="The managed System.Private.CoreLib library for CoreCLR." />
+    <SubsetName Include="NativeCoreLib" Category="CoreClr" Description="Run crossgen on System.Private.CoreLib library for CoreCLR." />
+    <SubsetName Include="Tools" Category="CoreClr" Description="Managed tools that support CoreCLR development and testing." />
+    <SubsetName Include="Packages" Category="CoreClr" Description="The projects that produce NuGet packages for the CoreCLR runtime, crossgen, and IL tools." />
 
     <!-- Mono -->
     <SubsetName Include="All" Category="Mono" Description="The Mono .NET runtime." />
@@ -98,14 +105,26 @@
   </ItemGroup>
 
   <!-- Global sets -->
-  <ItemGroup Condition="$(_subset.Contains('regeneratereadmetable'))">
+  <ItemGroup Condition="$(_subset.Contains('-regeneratereadmetable-'))">
     <ProjectToBuild Include="$(RepoToolsLocalDir)regenerate-readme-table.proj" />
   </ItemGroup>
 
   <ItemDefinitionGroup Condition="'$(CoreCLRConfiguration)' != ''">
     <CoreClrProjectToBuild>
       <AdditionalProperties>Configuration=$(CoreCLRConfiguration)</AdditionalProperties>
     </CoreClrProjectToBuild>
+    <CoreClrCoreLibProjectToBuild>
+      <AdditionalProperties>Configuration=$(CoreCLRConfiguration)</AdditionalProperties>
+    </CoreClrCoreLibProjectToBuild>
+    <CoreClrNativeCoreLibProjectToBuild>
+      <AdditionalProperties>Configuration=$(CoreCLRConfiguration)</AdditionalProperties>
+    </CoreClrNativeCoreLibProjectToBuild>
+    <CoreClrManagedToolsProjectToBuild>
+      <AdditionalProperties>Configuration=$(CoreCLRConfiguration)</AdditionalProperties>
+    </CoreClrManagedToolsProjectToBuild>
+    <CoreClrPackagesProjectToBuild>
+      <AdditionalProperties>Configuration=$(CoreCLRConfiguration)</AdditionalProperties>
+    </CoreClrPackagesProjectToBuild>
   </ItemDefinitionGroup>
 
   <ItemDefinitionGroup Condition="'$(MonoConfiguration)' != ''">
@@ -121,58 +140,80 @@
   </ItemDefinitionGroup>
 
   <!-- CoreClr sets -->
-  <ItemGroup Condition="$(_subsetCategory.Contains('coreclr')) and $(_subset.Contains('all'))">
-    <CoreClrProjectToBuild Include="$(CoreClrProjectRoot)coreclr.proj" BuildInParallel="false" />
+  <ItemGroup Condition="$(_subsetCategory.Contains('coreclr')) and $(_subset.Contains('-corelib-'))">
+    <CoreClrCoreLibProjectToBuild Include="$(CoreClrProjectRoot)src\System.Private.CoreLib\System.Private.CoreLib.csproj" BuildInParallel="false" />
+    <ProjectToBuild Include="@(CoreClrCoreLibProjectToBuild)" />
+  </ItemGroup>
+
+  <ItemGroup Condition="$(_subsetCategory.Contains('coreclr')) and $(_subset.Contains('-runtime-'))">
+    <CoreClrProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" BuildInParallel="false" />
     <ProjectToBuild Include="@(CoreClrProjectToBuild)" />
   </ItemGroup>
 
+  <ItemGroup Condition="$(_subsetCategory.Contains('coreclr')) and $(_subset.Contains('-nativecorelib-'))">
+    <CoreClrNativeCoreLibProjectToBuild Include="$(CoreClrProjectRoot)crossgen-corelib.proj" BuildInParallel="false" />
+    <ProjectToBuild Include="@(CoreClrNativeCoreLibProjectToBuild)" />
+  </ItemGroup>
+
+  <ItemGroup Condition="$(_subsetCategory.Contains('coreclr')) and $(_subset.Contains('-tools-'))">
+    <CoreClrManagedToolsProjectToBuild Include="$(CoreClrProjectRoot)src\tools\runincontext\runincontext.csproj" />
+    <CoreClrManagedToolsProjectToBuild Include="$(CoreClrProjectRoot)src\tools\r2rdump\R2RDump.csproj" />
+    <CoreClrManagedToolsProjectToBuild Include="$(CoreClrProjectRoot)src\tools\ReadyToRun.SuperIlc\ReadyToRun.SuperIlc.csproj" />
+    <CoreClrManagedToolsProjectToBuild Include="$(CoreClrProjectRoot)src\tools\crossgen2\crossgen2\crossgen2.csproj" BuildInParallel="false" />
+    <ProjectToBuild Include="@(CoreClrManagedToolsProjectToBuild)" />
+  </ItemGroup>
+
+  <ItemGroup Condition="$(_subsetCategory.Contains('coreclr')) and $(_subset.Contains('-packages-'))">
+    <CoreClrPackagesProjectToBuild Include="$(CoreClrProjectRoot)src\.nuget\packages.builds" BuildInParallel="false" />
+    <ProjectToBuild Include="@(CoreClrPackagesProjectToBuild)" />
+  </ItemGroup>
   <!-- Mono sets -->
-  <ItemGroup Condition="$(_subsetCategory.Contains('mono')) and $(_subset.Contains('all'))">
+  <ItemGroup Condition="$(_subsetCategory.Contains('mono')) and $(_subset.Contains('-all-'))">
     <MonoProjectToBuild Include="$(MonoProjectRoot)mono.proj" BuildInParallel="false" />
     <MonoProjectToBuild Include="$(MonoProjectRoot)netcore\System.Private.CoreLib\System.Private.CoreLib.csproj" BuildInParallel="false" />
     <ProjectToBuild Include="@(MonoProjectToBuild)" />
   </ItemGroup>
 
   <!-- Libraries sets -->
-  <ItemGroup Condition="$(_subsetCategory.Contains('libraries')) and $(_subset.Contains('all'))">
+  <ItemGroup Condition="$(_subsetCategory.Contains('libraries')) and $(_subset.Contains('-all-'))">
     <LibrariesProjectToBuild Include="$(LibrariesProjectRoot)build.proj" BuildInParallel="false" />
     <ProjectToBuild Include="@(LibrariesProjectToBuild)" />
   </ItemGroup>
 
   <!-- Installer sets -->
-  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('corehost'))">
+  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('-corehost-'))">
     <CorehostProjectToBuild Include="$(InstallerProjectRoot)corehost\build.proj" SignPhase="Binaries" />
     <ProjectToBuild Include="@(CorehostProjectToBuild)" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('managed'))">
+  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('-managed-'))">
     <ManagedProjectToBuild Include="$(InstallerProjectRoot)managed\**\*.csproj" SignPhase="Binaries" />
     <ManagedProjectToBuild Include="$(InstallerProjectRoot)pkg\packaging\pack-managed.proj" />
     <ProjectToBuild Include="@(ManagedProjectToBuild)" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('depproj'))">
+  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('-depproj-'))">
     <DepprojProjectToBuild Include="$(InstallerProjectRoot)pkg\projects\**\*.depproj" SignPhase="R2RBinaries" BuildInParallel="false" />
     <ProjectToBuild Include="@(DepprojProjectToBuild)" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('pkgproj'))">
+  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('-pkgproj-'))">
     <PkgprojProjectToBuild Include="$(InstallerProjectRoot)pkg\projects\**\*.pkgproj" SignPhase="MsiFiles" BuildInParallel="false" />
     <ProjectToBuild Include="@(PkgprojProjectToBuild)" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('bundle'))">
+  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('-bundle-'))">
     <BundleProjectToBuild Include="$(InstallerProjectRoot)pkg\projects\**\*.bundleproj" SignPhase="BundleInstallerFiles" BuildInParallel="false" />
     <ProjectToBuild Include="@(BundleProjectToBuild)" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('installers'))">
+  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('-installers-'))">
     <InstallerProjectToBuild Include="$(InstallerProjectRoot)pkg\packaging\installers.proj" BuildInParallel="false" />
     <InstallerProjectToBuild Include="$(InstallerProjectRoot)pkg\packaging\vs-insertion-packages.proj" BuildInParallel="false" />
     <ProjectToBuild Include="@(InstallerProjectToBuild)" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('test'))">
+  <ItemGroup Condition="$(_subsetCategory.Contains('installer')) and $(_subset.Contains('-test-'))">
     <TestProjectToBuild Include="$(InstallerProjectRoot)test\Microsoft.NET.HostModel.Tests\AppHost.Bundle.Tests\AppHost.Bundle.Tests.csproj" />
     <TestProjectToBuild Include="$(InstallerProjectRoot)test\Microsoft.NET.HostModel.Tests\Microsoft.NET.HostModel.AppHost.Tests\Microsoft.NET.HostModel.AppHost.Tests.csproj" />
     <TestProjectToBuild Include="$(InstallerProjectRoot)test\Microsoft.NET.HostModel.Tests\Microsoft.NET.HostModel.Bundle.Tests\Microsoft.NET.HostModel.Bundle.Tests.csproj" />

eng/build.ps1

@@ -11,7 +11,7 @@ Param(
   [switch]$allconfigurations,
   [switch]$coverage,
   [string]$testscope,
-  [string]$arch,
+  [string]$arch = [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant(),
   [string]$subsetCategory,
   [string]$subset,
   [ValidateSet("Debug","Release","Checked")][string]$runtimeConfiguration,
@@ -146,11 +146,13 @@ foreach ($argument in $PSBoundParameters.Keys)
     "framework"            { $arguments += " /p:BuildTargetFramework=$($PSBoundParameters[$argument].ToLowerInvariant())" }
     "os"                   { $arguments += " /p:OSGroup=$($PSBoundParameters[$argument])" }
     "allconfigurations"    { $arguments += " /p:BuildAllConfigurations=true" }
-    "arch"                 { $arguments += " /p:ArchGroup=$($PSBoundParameters[$argument]) /p:TargetArchitecture=$($PSBoundParameters[$argument])" }
+    "arch"                 { $arch = $PSBoundParameters[$argument]; $arguments += " /p:ArchGroup=$($PSBoundParameters[$argument]) /p:TargetArchitecture=$($PSBoundParameters[$argument])" }
     "properties"           { $arguments += " " + $properties }
     default                { $arguments += " /p:$argument=$($PSBoundParameters[$argument])" }
   }
 }
 
+$env:__DistroRid="win-$arch"
+
 Invoke-Expression "& `"$PSScriptRoot/common/build.ps1`" $arguments"
 exit $lastExitCode

eng/build.sh

@@ -60,13 +60,35 @@ usage()
   echo "Arguments can also be passed in with a single hyphen."
 }
 
+initDistroRid()
+{
+    source $scriptroot/native/init-distro-rid.sh
+
+    local passedRootfsDir=""
+    local buildOs="$1"
+    local buildArch="$2"
+    local isCrossBuild="$3"
+    # For RID calculation purposes, say we are always a portable build
+    # All of our packages that use the distro rid (CoreCLR packages) are portable.
+    local isPortableBuild=1
+
+    # Only pass ROOTFS_DIR if __DoCrossArchBuild is specified.
+    if (( isCrossBuild == 1 )); then
+        passedRootfsDir=${ROOTFS_DIR}
+    fi
+    initDistroRidGlobal ${buildOs} ${buildArch} ${isPortableBuild} ${passedRootfsDir}
+}
+
 arguments=''
 cmakeargs=''
 extraargs=''
 build=false
 buildtests=false
 subsetCategory=''
 checkedPossibleDirectoryToBuild=false
+crossBuild=0
+
+source $scriptroot/native/init-os-and-arch.sh
 
 # Check if an action is passed in
 declare -a actions=("r" "restore" "b" "build" "buildtests" "rebuild" "t" "test" "pack" "sign" "publish" "clean")
@@ -89,6 +111,7 @@ while [[ $# > 0 ]]; do
       shift 2
       ;;
      -arch)
+      arch=$2
       arguments="$arguments /p:ArchGroup=$2 /p:TargetArchitecture=$2"
       shift 2
       ;;
@@ -103,6 +126,7 @@ while [[ $# > 0 ]]; do
       shift 2
       ;;
      -os)
+      os=$2
       arguments="$arguments /p:OSGroup=$2"
       shift 2
       ;;
@@ -141,6 +165,7 @@ while [[ $# > 0 ]]; do
       shift 2
       ;;
      -cross)
+      crossBuild=1
       arguments="$arguments /p:CrossBuild=True"
       shift 1
       ;;
@@ -187,6 +212,8 @@ if [ ${#actInt[@]} -eq 0 ]; then
     arguments="-restore -build $arguments"
 fi
 
+initDistroRid $os $arch $crossBuild
+
 # URL-encode space (%20) to avoid quoting issues until the msbuild call in /eng/common/tools.sh.
 # In *proj files (XML docs), URL-encoded string are rendered in their decoded form.
 cmakeargs="${cmakeargs// /%20}"

eng/native/build-commons.sh

@@ -193,92 +193,12 @@ usage()
     exit 1
 }
 
-# Use uname to determine what the CPU is.
-CPUName=$(uname -p)
+source "$__RepoRootDir/eng/native/init-os-and-arch.sh"
 
-# Some Linux platforms report unknown for platform, but the arch for machine.
-if [[ "$CPUName" == "unknown" ]]; then
-    CPUName=$(uname -m)
-fi
-
-case "$CPUName" in
-    aarch64)
-        __BuildArch=arm64
-        __HostArch=arm64
-        ;;
-
-    amd64)
-        __BuildArch=x64
-        __HostArch=x64
-        ;;
-
-    armv7l)
-        if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then
-            __BuildArch=armel
-            __HostArch=armel
-        else
-            __BuildArch=arm
-            __HostArch=arm
-        fi
-        ;;
-
-    i686)
-        echo "Unsupported CPU $CPUName detected, build might not succeed!"
-        __BuildArch=x86
-        __HostArch=x86
-        ;;
-
-    x86_64)
-        __BuildArch=x64
-        __HostArch=x64
-        ;;
-
-    *)
-        echo "Unknown CPU $CPUName detected, configuring as if for x64"
-        __BuildArch=x64
-        __HostArch=x64
-        ;;
-esac
-
-# Use uname to determine what the OS is.
-OSName=$(uname -s)
-case "$OSName" in
-    Darwin)
-        __BuildOS=OSX
-        __HostOS=OSX
-        ;;
-
-    FreeBSD)
-        __BuildOS=FreeBSD
-        __HostOS=FreeBSD
-        ;;
-
-    Linux)
-        __BuildOS=Linux
-        __HostOS=Linux
-        ;;
-
-    NetBSD)
-        __BuildOS=NetBSD
-        __HostOS=NetBSD
-        ;;
-
-    OpenBSD)
-        __BuildOS=OpenBSD
-        __HostOS=OpenBSD
-        ;;
-
-    SunOS)
-        __BuildOS=SunOS
-        __HostOS=SunOS
-        ;;
-
-    *)
-        echo "Unsupported OS $OSName detected, configuring as if for Linux"
-        __BuildOS=Linux
-        __HostOS=Linux
-        ;;
-esac
+__BuildArch=$arch
+__HostArch=$arch
+__BuildOS=$os
+__HostOS=$os
 
 __msbuildonunsupportedplatform=0