Script changes enabling composite R2R build with shared framework

eng/pipelines/coreclr/crossgen2-composite.yml

@@ -0,0 +1,56 @@
+trigger: none
+
+pr: none
+
+schedules:
+- cron: "0 6 * * *"
+  displayName: Mon through Sun at 10:00 PM (UTC-8:00)
+  branches:
+    include:
+    - master
+  always: true
+
+jobs:
+#
+# Checkout repository
+#
+- template: /eng/pipelines/common/checkout-job.yml
+
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
+    buildConfig: checked
+    platforms:
+    - Linux_x64
+    - OSX_x64
+    - Windows_NT_x64
+    jobParameters:
+      testGroup: innerloop
+
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml
+    buildConfig: checked
+    platforms:
+    - OSX_x64
+    - Windows_NT_x64
+    jobParameters:
+      testGroup: innerloop
+      liveLibrariesBuildConfig: Release
+
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+    buildConfig: checked
+    platforms:
+    - Linux_x64
+    - OSX_x64
+    - Windows_NT_x64
+    jobParameters:
+      testGroup: innerloop
+      readyToRun: true
+      crossgen2: true
+      compositeBuildMode: true
+      displayNameArgs: Composite
+      liveLibrariesBuildConfig: Release

eng/pipelines/coreclr/templates/run-test-job.yml

@@ -11,6 +11,7 @@ parameters:
   readyToRun: false
   liveLibrariesBuildConfig: ''
   crossgen2: false
+  compositeBuildMode: false
   helixQueues: ''
   # If true, run the corefx tests instead of the coreclr ones
   corefxTests: false
@@ -91,6 +92,9 @@ jobs:
           value: 'crossgen2'
         - name: LogNamePrefix
           value: TestRunLogs_R2R_CG2
+      - ${{ if eq(parameters.compositeBuildMode, true) }}:
+        - name: crossgenArg
+          value: 'composite'
 
     # Set job timeouts
     #
@@ -141,6 +145,10 @@ jobs:
       - name: timeoutPerTestInMinutes
         value: 90
 
+    - ${{ if eq(parameters.compositeBuildMode, true) }}:
+      - name: crossgenArg
+        value: 'composite'
+
     - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
       - group: DotNet-HelixApi-Access
 
@@ -277,6 +285,7 @@ jobs:
 
         runCrossGen: ${{ and(eq(parameters.readyToRun, true), ne(parameters.crossgen2, true)) }}
         runCrossGen2: ${{ and(eq(parameters.readyToRun, true), eq(parameters.crossgen2, true)) }}
+        compositeBuildMode: ${{ parameters.compositeBuildMode }}
         runInUnloadableContext: ${{ parameters.runInUnloadableContext }}
 
         ${{ if eq(variables['System.TeamProject'], 'internal') }}:

eng/pipelines/coreclr/templates/send-to-helix-step.yml

@@ -16,6 +16,7 @@ parameters:
   timeoutPerTestInMinutes: ''
   runCrossGen: ''
   runCrossGen2: ''
+  compositeBuildMode: false
   helixProjectArguments: ''
   runInUnloadableContext: ''
   longRunningGcTests: ''
@@ -47,6 +48,7 @@ steps:
       _HelixType: ${{ parameters.helixType }}
       _RunCrossGen: ${{ parameters.runCrossGen }}
       _RunCrossGen2: ${{ parameters.runCrossGen2 }}
+      _CompositeBuildMode: ${{ parameters.compositeBuildMode }}
       _RunInUnloadableContext: ${{ parameters.runInUnloadableContext }}
       _LongRunningGcTests: ${{ parameters.longRunningGcTests }}
       _GcSimulatorTests: ${{ parameters.gcSimulatorTests }}
@@ -86,6 +88,7 @@ steps:
       _HelixType: ${{ parameters.helixType }}
       _RunCrossGen: ${{ parameters.runCrossGen }}
       _RunCrossGen2: ${{ parameters.runCrossGen2 }}
+      _CompositeBuildMode: ${{ parameters.compositeBuildMode }}
       _RunInUnloadableContext: ${{ parameters.runInUnloadableContext }}
       _LongRunningGcTests: ${{ parameters.longRunningGcTests }}
       _GcSimulatorTests: ${{ parameters.gcSimulatorTests }}

src/coreclr/build-test.cmd

@@ -59,6 +59,7 @@ set __RuntimeId=
 set __TargetsWindows=1
 set __DoCrossgen=
 set __DoCrossgen2=
+set __CompositeBuildMode=
 set __CopyNativeTestBinaries=0
 set __CopyNativeProjectsAfterCombinedTestBuild=true
 set __SkipGenerateLayout=0
@@ -106,6 +107,7 @@ if /i "%1" == "buildagainstpackages"  (echo error: Remove /BuildAgainstPackages
 if /i "%1" == "skiprestorepackages"   (set __SkipRestorePackages=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "crossgen"              (set __DoCrossgen=1&set __TestBuildMode=crossgen&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "crossgen2"             (set __DoCrossgen2=1&set __TestBuildMode=crossgen2&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "composite"             (set __CompositeBuildMode=1&set __DoCrossgen2=1&set __TestBuildMode=crossgen2&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "runtimeid"             (set __RuntimeId=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
 if /i "%1" == "targetsNonWindows"     (set __TargetsWindows=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "Exclude"               (set __Exclude=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
@@ -627,18 +629,45 @@ exit /b 1
 set __TotalPrecompiled=0
 set __FailedToPrecompile=0
 set __FailedAssemblies=
-for %%F in ("%CORE_ROOT%\System.*.dll";"%CORE_ROOT%\Microsoft.*.dll") do (
+set __CompositeOutputDir=%CORE_ROOT%\composite.out
+set __CompositeResponseFile=%__CompositeOutputDir%\framework-r2r.dll.rsp
+
+if defined __CompositeBuildMode (
+    mkdir !__CompositeOutputDir!
+    echo --composite>>!__CompositeResponseFile!
+    echo -O>>!__CompositeResponseFile!
+    echo --out^:%__CompositeOutputDir%\framework-r2r.dll>>!__CompositeResponseFile!
+)
+
+for %%F in ("%CORE_ROOT%\System.*.dll";"%CORE_ROOT%\Microsoft.*.dll";%CORE_ROOT%\netstandard.dll;%CORE_ROOT%\mscorlib.dll) do (
     if not "%%~nxF"=="Microsoft.CodeAnalysis.VisualBasic.dll" (
     if not "%%~nxF"=="Microsoft.CodeAnalysis.CSharp.dll" (
     if not "%%~nxF"=="Microsoft.CodeAnalysis.dll" (
     if not "%%~nxF"=="System.Runtime.WindowsRuntime.dll" (
-        call :PrecompileAssembly "%%F" %%~nxF __TotalPrecompiled __FailedToPrecompile __FailedAssemblies
-        echo Processed: !__TotalPrecompiled!, failed !__FailedToPrecompile!
+        if defined __CompositeBuildMode (
+            echo %%F>>!__CompositeResponseFile!
+        ) else (
+            call :PrecompileAssembly "%%F" %%~nxF __TotalPrecompiled __FailedToPrecompile __FailedAssemblies
+            echo Processed: !__TotalPrecompiled!, failed !__FailedToPrecompile!
+        )
     )))))
 )
 
+echo Composite response line^: %__CompositeResponseFile%
+type "%__CompositeResponseFile%"
+
+if defined __CompositeBuildMode (
+    set __CompositeCommandLine="%CORE_ROOT%\corerun"
+    set __CompositeCommandLine=!__CompositeCommandLine! "%CORE_ROOT%\crossgen2\crossgen2.dll"
+    set __CompositeCommandLine=!__CompositeCommandLine! "@%__CompositeResponseFile%"
+    echo Building composite R2R framework^: !__CompositeCommandLine!
+    !__CompositeCommandLine!
+    set __FailedToPrecompile=!ERRORLEVEL!
+    copy /Y "!__CompositeOutputDir!\*.*" "!CORE_ROOT!\"
+)
+
 if !__FailedToPrecompile! NEQ 0 (
-    echo Failed assemblies:
+    @echo Failed assemblies:
     FOR %%G IN (!__FailedAssemblies!) do echo   %%G
 )
 

src/coreclr/build-test.sh

@@ -183,15 +183,25 @@ precompile_coreroot_fx()
 
     local totalPrecompiled=0
     local failedToPrecompile=0
+    local compositeCommandLine="$overlayDir/corerun"
+    compositeCommandLine+=" $overlayDir/crossgen2/crossgen2.dll"
+    compositeCommandLine+=" --composite"
+    compositeCommandLine+=" -O"
+    compositeCommandLine+=" --out:$outputDir/framework-r2r.dll"
     declare -a failedAssemblies
 
-    filesToPrecompile=$(find -L "$overlayDir" -maxdepth 1 -iname Microsoft.\*.dll -o -iname System.\*.dll -type f)
+    filesToPrecompile=$(find -L "$overlayDir" -maxdepth 1 -iname Microsoft.\*.dll -o -iname System.\*.dll -o -iname netstandard.dll -o -iname mscorlib.dll -type f)
     for fileToPrecompile in ${filesToPrecompile}; do
         local filename="$fileToPrecompile"
         if is_skip_crossgen_test "$(basename $filename)"; then
             continue
         fi
 
+        if [[ "$__CompositeBuildMode" != 0 ]]; then
+            compositeCommandLine+=" $filename"
+            continue
+        fi
+
         local commandLine=""
 
         if [[ "$__DoCrossgen" != 0 ]]; then
@@ -224,6 +234,17 @@ precompile_coreroot_fx()
         echo "Processed: $totalPrecompiled, failed $failedToPrecompile"
     done
 
+    if [[ "$__CompositeBuildMode" != 0 ]]; then
+        # Compile the entire framework in composite build mode
+        echo "Compiling composite R2R framework: $compositeCommandLine"
+        $compositeCommandLine
+        local exitCode="$?"
+        if [[ "$exitCode" != 0 ]]; then
+            echo Unable to precompile composite framework, exit code is "$exitCode".
+            exit 1
+        fi
+    fi
+
     if [[ "$__DoCrossgen2" != 0 ]]; then
         # Copy the Crossgen-compiled assemblies back to CORE_ROOT
         mv -f "$outputDir"/* "$overlayDir"/
@@ -536,6 +557,12 @@ handle_arguments_local() {
             __TestBuildMode=crossgen2
             ;;
 
+        composite|-composite)
+            __CompositeBuildMode=1
+            __DoCrossgen2=1
+            __TestBuildMode=crossgen2
+            ;;
+
         generatetesthostonly|-generatetesthostonly)
             __GenerateTestHostOnly=1
             ;;
@@ -603,6 +630,7 @@ __CrossBuild=0
 __DistroRid=""
 __DoCrossgen=0
 __DoCrossgen2=0
+__CompositeBuildMode=0
 __DotNetCli="$__RepoRootDir/dotnet.sh"
 __GenerateLayoutOnly=
 __GenerateTestHostOnly=

src/coreclr/tests/helixpublishwitharcade.proj

@@ -27,6 +27,7 @@
         PublishTestResults=$(_PublishTestResults);
         RunCrossGen=$(_RunCrossGen);
         RunCrossGen2=$(_RunCrossGen2);
+        CompositeBuildMode=$(_CompositeBuildMode);
         LongRunningGCTests=$(_LongRunningGCTests);
         GcSimulatorTests=$(_GcSimulatorTests);
         RunInUnloadableContext=$(_RunInUnloadableContext);
@@ -194,6 +195,7 @@
     <HelixPreCommand Include="set _NT_SYMBOL_PATH=%HELIX_CORRELATION_PAYLOAD%\PDB" />
     <HelixPreCommand Include="set RunCrossGen=1" Condition=" '$(RunCrossGen)' == 'true' " />
     <HelixPreCommand Include="set RunCrossGen2=1" Condition=" '$(RunCrossGen2)' == 'true' " />
+    <HelixPreCommand Include="set CompositeBuildMode=1" Condition=" '$(CompositeBuildMode)' == 'true' " />
     <HelixPreCommand Include="set RunningLongGCTests=1" Condition=" '$(LongRunningGCTests)' == 'true' " />
     <HelixPreCommand Include="set RunningGCSimulatorTests=1" Condition=" '$(GcSimulatorTests)' == 'true' " />
     <HelixPreCommand Include="set RunInUnloadableContext=1" Condition=" '$(RunInUnloadableContext)' == 'true' " />
@@ -209,6 +211,7 @@
     <HelixPreCommand Include="export CORE_ROOT=$HELIX_CORRELATION_PAYLOAD" />
     <HelixPreCommand Include="export RunCrossGen=1" Condition=" '$(RunCrossGen)' == 'true' " />
     <HelixPreCommand Include="export RunCrossGen2=1" Condition=" '$(RunCrossGen2)' == 'true' " />
+    <HelixPreCommand Include="export CompositeBuildMode=1" Condition=" '$(CompositeBuildMode)' == 'true' " />
     <HelixPreCommand Include="export RunningLongGCTests=1" Condition=" '$(LongRunningGCTests)' == 'true' " />
     <HelixPreCommand Include="export RunningGCSimulatorTests=1" Condition=" '$(GcSimulatorTests)' == 'true' " />
     <HelixPreCommand Include="export RunInUnloadableContext=1" Condition=" '$(RunInUnloadableContext)' == 'true' " />