Merge pull request #34166 from mangod9/master

Add support for building S.P.CoreLib in a separate job

eng/pipelines/common/build-coreclr-and-libraries-job.yml

@@ -13,6 +13,21 @@ parameters:
   pool: ''
 
 jobs:
+- template: /eng/pipelines/coreclr/templates/build-corelib-job.yml
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: ${{ parameters.archType }}
+    osGroup: ${{ parameters.osGroup }}
+    osSubgroup: ${{ parameters.osSubgroup }}
+    container: ${{ parameters.container }}
+    testGroup: ${{ parameters.testGroup }}
+    crossrootfsDir: ${{ parameters.crossrootfsDir }}
+    timeoutInminutes: ${{ parameters.timeoutInMinutes }}
+    signBinaries: ${{ parameters.signBinaries }}
+    stagedBuild: ${{ parameters.stagedBuild }}
+    variables: ${{ parameters.variables }}
+    pool: ${{ parameters.pool }}
+
 - template: /eng/pipelines/coreclr/templates/build-job.yml
   parameters:
     buildConfig: ${{ parameters.buildConfig }}

eng/pipelines/coreclr/ci.yml

@@ -31,6 +31,17 @@ jobs:
 #
 - template: /eng/pipelines/common/checkout-job.yml
 
+#
+# Checked Corelib builds
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml
+    buildConfig: checked
+    platformGroup: all
+    jobParameters:
+      testGroup: outerloop
+
 #
 # Debug builds
 #

eng/pipelines/coreclr/templates/build-corelib-job.yml

@@ -0,0 +1,109 @@
+parameters:
+  archType: ''
+  buildConfig: ''
+  condition: true
+  container: ''
+  crossrootfsDir: ''
+  isOfficialBuild: false
+  osGroup: ''
+  osSubgroup: ''
+  platform: ''
+  pool: ''
+  signBinaries: false
+  stagedBuild: false
+  testGroup: ''
+  timeoutInMinutes: ''
+  variables: {}
+
+### Product build
+jobs:
+- template: xplat-pipeline-job.yml
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    _BuildConfig: ${{ parameters.buildConfig }}
+    archType: ${{ parameters.archType }}
+    osGroup: ${{ parameters.osGroup }}
+    osSubgroup: ${{ parameters.osSubgroup }}
+    testGroup: ${{ parameters.testGroup }}
+    helixType: 'build/product/'
+    enableMicrobuild: true
+    stagedBuild: ${{ parameters.stagedBuild }}
+    pool: ${{ parameters.pool }}
+    condition: ${{ parameters.condition }}
+
+    name: ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+    displayName: ${{ format('CoreCLR CoreLib Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+
+    # Run all steps in the container.
+    # Note that the containers are defined in platform-matrix.yml
+    container: ${{ parameters.container }}
+
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
+    crossrootfsDir: ${{ parameters.crossrootfsDir }}
+
+    gatherAssetManifests: true
+    variables:
+    - name: osGroup
+      value: ${{ parameters.osGroup }}
+    - name: osSubgroup
+      value: ${{ parameters.osSubgroup }}
+    - name: compilerArg
+      value: ''
+    - name: publishLogsArtifactPrefix
+      value: 'BuildLogs_CoreLib'
+    - name: officialBuildIdArg
+      value: ''
+    - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
+      - name: officialBuildIdArg
+        value: '/p:OfficialBuildId=$(Build.BuildNumber)'
+
+    - ${{ parameters.variables }}
+
+    steps:
+
+    # Install native dependencies
+    # Linux builds use docker images with dependencies preinstalled,
+    # and FreeBSD builds use a build agent with dependencies
+    # preinstalled, so we only need this step for OSX and Windows.
+    - ${{ if eq(parameters.osGroup, 'OSX') }}:
+      - script: sh $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup)
+        displayName: Install native dependencies
+    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+      # Necessary to install python
+      - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
+        displayName: Install native dependencies
+
+    # # Install internal tools on official builds
+    # # Since our internal tools are behind an authenticated feed,
+    # # we need to use the DotNetCli AzDO task to restore from the feed using a service connection.
+    # # We can't do this from within the build, so we need to do this as a separate step.
+    - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
+      - template: /eng/pipelines/common/restore-internal-tools.yml
+        parameters:
+          installDotnet: true
+
+    # Build Private CoreLib
+    - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subsetCategory coreclr -subset corelib $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci
+      displayName: Build System.Private.CoreLib
+
+
+    # Publish corelib output directory for consumption by libraries.
+    - template: /eng/pipelines/common/upload-artifact-step.yml
+      parameters:
+        rootFolder: $(buildProductRootFolderPath)
+        includeRootFolder: false
+        archiveType: $(archiveType)
+        tarCompression: $(tarCompression)
+        archiveExtension: $(archiveExtension)
+        artifactName: $(corelibProductArtifactName)
+        displayName: Upload System.Private.CoreLib
+
+    # Publish Logs
+    - task: PublishPipelineArtifact@1
+      displayName: Publish Logs
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/artifacts/log
+        artifactName: '$(publishLogsArtifactPrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+      continueOnError: true
+      condition: always()

eng/pipelines/coreclr/templates/xplat-pipeline-job.yml

@@ -75,6 +75,9 @@ jobs:
     - name: buildProductRootFolderPath
       value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)'
 
+    - name: corelibProductArtifactName
+      value: 'CoreLib_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+
     - name: managedTestArtifactName
       value: 'CoreCLRManagedTestArtifacts_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_$(archType)_$(buildConfig)'
 

eng/pipelines/libraries/base-job.yml

@@ -99,7 +99,12 @@ jobs:
 
         - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
           - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}'
-          - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
+          # Download corelib dependencies for coreclr libraries and not test
+          - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}:
+            - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
+          # Download full product dependencies for mono or test
+          - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}:
+            - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
           - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)'
           - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }}
 

eng/pipelines/libraries/build-job.yml

@@ -37,6 +37,7 @@ jobs:
       isOfficialBuild: ${{ parameters.isOfficialBuild }}
       isOfficialAllConfigurations: ${{ parameters.isOfficialAllConfigurations }}
       liveRuntimeBuildConfig: ${{ parameters.liveRuntimeBuildConfig }}
+      runtimeFlavor: ${{ parameters.runtimeFlavor }}
       timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
       preBuildSteps: ${{ parameters.preBuildSteps }}
       container: ${{ parameters.container }}
@@ -49,7 +50,11 @@ jobs:
       ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
         dependsOn:
         - ${{ if ne(parameters.osGroup, 'WebAssembly') }}:
-          - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
+          # Use corelib dependencies for coreclr and non test builds
+          - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}:
+            - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
+          - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}:
+            - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
         - ${{ if eq(parameters.osGroup, 'WebAssembly') }}:
           - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }}
 

eng/pipelines/libraries/build-test-job.yml

@@ -6,6 +6,7 @@ parameters:
   framework: ''
   isOfficialBuild: false
   liveRuntimeBuildConfig: ''
+  runtimeFlavor: 'coreclr'
   timeoutInMinutes: 150
   container: ''
   publishTestArtifacs: true
@@ -38,6 +39,9 @@ jobs:
         - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
       - ${{ if in(parameters.framework, 'allConfigurations', 'net472') }}:
         - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+      # Libraries Test also depends on Product, now that the libraries build only depends on corelib
+      - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
+        - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
 
       variables:
         - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}

eng/pipelines/libraries/outerloop.yml

@@ -14,6 +14,29 @@ variables:
 jobs:
   - template: /eng/pipelines/common/checkout-job.yml
 
+  #
+  # CoreLib Build
+  #
+  - template: /eng/pipelines/common/platform-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml
+      buildConfig: release
+      platforms:
+      - ${{ if eq(variables['includeWindowsOuterloop'], true) }}:
+        - Windows_NT_x64
+        - Windows_NT_x86
+      - ${{ if eq(variables['includeLinuxOuterloop'], true) }}:
+        - Linux_x64
+        - Linux_musl_x64
+        - ${{ if eq(variables['isFullMatrix'], true) }}:
+          - Linux_arm
+          - Linux_arm64
+          - Linux_musl_arm64
+      - ${{ if eq(variables['includeOsxOuterloop'], true) }}:        
+        - OSX_x64
+      jobParameters:
+        testGroup: innerloop
+
   #
   # CoreCLR Build
   #

eng/pipelines/libraries/run-test-job.yml

@@ -31,6 +31,7 @@ jobs:
       container: '' # we just send to helix, no need to use a container.
       condition: ${{ parameters.condition }}
       pool: ${{ parameters.pool }}
+      testScope: ${{ parameters.testScope }}
       ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
         displayName: ${{ format('Test Run {0} {1}', parameters.liveRuntimeBuildConfig, parameters.runtimeFlavor) }}
         name: ${{ format('test_run_{0}_{1}', parameters.liveRuntimeBuildConfig, parameters.runtimeFlavor) }}
@@ -78,9 +79,19 @@ jobs:
                     /p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
                     /p:TargetArchitecture=${{ parameters.archType }}
                     /p:Configuration=${{ parameters.buildConfig }}
-                    /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideCoreclrFromLiveDrop.binlog
+                    /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideRuntimeFromLiveDrop.binlog
             displayName: Override $(runtimeFlavorName) from live drop
 
+          - script: $(_buildScript) -projects $(Build.SourcesDirectory)/src/libraries/pretest.proj 
+                    $(_runtimeArtifactsPathArg)
+                    $(_runtimeOSArg)
+                    /p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
+                    /p:TargetArchitecture=${{ parameters.archType }}
+                    /p:Configuration=${{ parameters.buildConfig }}
+                    /p:SkipBuildProjects=true
+                    /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/pretest.binlog
+            displayName: Build PreTest to Generate Deps Json File
+
         - template: /eng/pipelines/libraries/helix.yml
           parameters:
             runtimeFlavor: ${{ parameters.runtimeFlavor }}

eng/pipelines/runtime-official.yml

@@ -39,6 +39,27 @@ stages:
   #
   - template: /eng/pipelines/common/checkout-job.yml
 
+  #
+  # Build CoreLib
+  #
+  - template: /eng/pipelines/common/platform-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml
+      buildConfig: release
+      platforms:
+      - OSX_x64
+      - Linux_x64
+      - Linux_arm
+      - Linux_arm64
+      - Linux_musl_x64
+      - Linux_musl_arm64
+      - Windows_NT_x86
+      - Windows_NT_x64
+      - Windows_NT_arm
+      - Windows_NT_arm64
+      jobParameters:
+        isOfficialBuild: ${{ variables.isOfficialBuild }}
+
   #
   # Build CoreCLR
   #

eng/pipelines/runtime.yml

@@ -118,6 +118,27 @@ jobs:
       - eng/pipelines/mono/*
       - eng/pipelines/libraries/*
 
+#
+# Build CoreLib release
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml
+    buildConfig: release
+    platforms:
+    - OSX_x64
+    - Linux_x64
+    - Linux_arm
+    - Linux_arm64
+    - Linux_musl_x64
+    - Linux_musl_arm64
+    - Windows_NT_x64
+    - Windows_NT_x86
+    - Windows_NT_arm
+    - Windows_NT_arm64
+    jobParameters:
+      testGroup: innerloop
+
 #
 # Build CoreCLR checked
 # Only when CoreCLR is changed

src/libraries/pretest.proj

@@ -78,7 +78,8 @@
              Projects="@(Project)"
              Properties="TargetFramework=$(BuildTargetFramework);%(Project.AdditionalProperties)"
              BuildInParallel="true"
-             ContinueOnError="ErrorAndStop" />
+             ContinueOnError="ErrorAndStop" 
+             Condition="'$(SkipBuildProjects)' != 'true'"/>
 
     <!-- Given we ErrorAndContinue we need to propagate the error if the overall task failed -->
     <Error Condition="'$(MSBuildLastTaskResult)' == 'false'" />

src/libraries/tests.proj

@@ -41,8 +41,19 @@
   <Import Project="$(RepositoryEngineeringDir)coverage.targets" Condition="'$(EnableCoverageSupport)' == 'true'" />
   <Import Sdk="Microsoft.NET.Sdk" Project="Sdk.targets" />
 
+  <ItemGroup>
+    <SetupProjects Include="$(MSBuildThisFileDirectory)restore\runtime\runtime.depproj" />
+    <SetupProjects Include="$(MSBuildThisFileDirectory)\pretest.proj" />
+  </ItemGroup>
+
   <Target Name="Build" DependsOnTargets="BuildAllProjects" />
   <Target Name="BuildAllProjects">
+    <!-- These projects are required to restore the runtime and setup testhost -->
+    <MSBuild Targets="Build"
+             Projects="@(SetupProjects)"
+             Condition="'$(DotNetBuildFromSource)' != 'true' and '$(BuildAllConfigurations)' != 'true' and '$(ContinuousIntegrationBuild)' == 'true'" 
+             BuildInParallel="false"
+             ContinueOnError="ErrorAndStop" />
     <MSBuild Targets="Build"
              Projects="@(Project)"
              Properties="BuildAllProjects=true;%(Project.AdditionalProperties)"