Update runtime build to include libraries and installer (#1576)

* Add runtime repo and remove coreclr, corefx and core-setup * Update arcade and standard shas * Update patches to enable runtime repo to build * Add patch for ilasm round trip issue * Add patch for ilasm round trip issue * Reconcile patches for runtime repo * Reconcile patches for arcade repo * Add patch to set nopgooptimize in coreclr build * Add CentOS 8 CI build (#1539) * Add CentOS 8 CI build We have had some issues building in RHEL 8, so lets set up a CentOS 8 CI system to try and catch those issues earlier. * Pin to artifacts version with executable coreclr bits Co-authored-by: Aditya Aggarwal <aditya.aggarwal@microsoft.com> Conflicts: eng/Versions.props repos/coreclr.common.props * Only build through coreclr for now * Include 1 addtional arcade patch * Add patch to exclude Helix tests from source-build in arcade * Add reference to PlatformAbstractions * Skip prebuilt check for now * Disable smoke-tests and unit tests until core-sdk build works * Update Centos8 docker image to one that includes updated cmake * Another try at updating the centos8 docker image * Revert "Skip prebuilt check for now" This reverts commit 5766e7e. * Add comments & issue links to disabled items * Update to document temporary smoke-test removal * Update build script to take additional params for all builds Add patch to not build harvestPackages in source-build Remove PVP from dependencies.props and eng/Versions.props * Don't default version of MsbuildTaskMicrosoftCodeAnalysisCSharpVersion in arcade * Update versions of arcade and SBRP * Add new sdk overrides * Re-enable installer build * Update version of source-built artifacts * Build crossgen 2 and update arcade patches * Update dotnet and source-built versions * don't override NuGetVersion in arcade * Temporary fix to save arcade specific prebuilts that are ref-pkgs * Update prebuilt baselines * Remove extra files from rebase * Remove extra centos8 job * Update portable runtime settings from PR#1530 in the release/3.1 branch * Updating allowed prebuilts * Patch to conditionally update CodeAnalysisCSharpVersion * Turn off prebuilt check. Issue to re-enable #1577 * Update portable build properties * One prebuilt update for osx * Updates from PR review * Remove extraneous space * One more extraneous space Co-authored-by: Omair Majid <omajid@redhat.com>
dotnet · May 7, 2020 · ecbdc42 · ecbdc42
1 parent d41441d
commit ecbdc42
Show file tree

Hide file tree

Showing 49 changed files with 689 additions and 258 deletions.
diff --git a/.vsts.pipelines/builds/matrix.yml b/.vsts.pipelines/builds/matrix.yml
@@ -23,7 +23,6 @@ jobs:
       Production: { generatePrebuiltBurndown: true, runUnitTests: false }
       Online: { type: Online }
       Offline: { type: Offline }
-      Offline Portable: { type: Offline Portable }
 
 # Temporarily disable bootstrap build until entire product builds.
 # See https://github.com/dotnet/source-build/issues/1550
@@ -39,12 +38,12 @@ jobs:
   parameters:
     job: centos8
     imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-source-build-20200402192642-9e679d4
+    reportPrebuiltLeaks: false
     systemLibunwind: false
     matrix:
       Production: {}
       Online: { type: Online }
       Offline: { type: Offline }
-      Offline Portable: { type: Offline Portable }
 
 - template: ../jobs/ci-linux.yml
   parameters:
@@ -72,7 +71,6 @@ jobs:
       Production: {}
       Online: { type: Online }
       Offline: { type: Offline }
-      Offline Portable: { type: Offline Portable }
 
 - template: ../jobs/ci-linux.yml
   parameters:

diff --git a/.vsts.pipelines/jobs/ci-linux.yml b/.vsts.pipelines/jobs/ci-linux.yml
@@ -66,7 +66,8 @@ jobs:
       fi
       $(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
         /p:Configuration=$(sb.configuration) \
-        /p:PortableBuild=$(sb.portable) \
+        /p:BuildPortableRuntime=$(sb.portable) \
+        /p:BuildPortableSdk=$(sb.portable) \
         /p:UseSystemLibunwind=$(systemLibunwind) \
         /p:ArchiveDownloadedPackages=$(sb.tarball) \
         /p:FailOnPrebuiltBaselineError=$failOnBaselineError \
@@ -117,7 +118,7 @@ jobs:
   - script: |
       set -ex
       df -h
-      args="--skip-build --minimize-disk-usage"
+      args="--skip-build --minimize-disk-usage --skip-prebuilt-check"
       if [ "$(reportPrebuiltLeaks)" = "true" ]; then
         args="$args --enable-leak-detection"
       fi
@@ -215,7 +216,8 @@ jobs:
       fi
       $(docker.run) $(docker.tb.map) $(docker.tb.work) $networkArg $(imageName) "$(tarballName)/build.sh" -- \
         /p:Configuration=$(sb.configuration) \
-        /p:PortableBuild=$(sb.portable) \
+        /p:BuildPortableRuntime=$(sb.portable) \
+        /p:BuildPortableSdk=$(sb.portable) \
         /p:UseSystemLibunwind=$(systemLibunwind) \
         /p:FailOnPrebuiltBaselineError=true \
         $poisonArg

diff --git a/.vsts.pipelines/jobs/ci-local.yml b/.vsts.pipelines/jobs/ci-local.yml
@@ -21,7 +21,8 @@ jobs:
   variables:
     args.build: >
       /p:Configuration=$(sb.configuration)
-      /p:PortableBuild=$(sb.portable)
+      /p:BuildPortableRuntime=$(sb.portable) \
+      /p:BuildPortableSdk=$(sb.portable) \
       /p:FailOnPrebuiltBaselineError=$(failOnPrebuiltBaselineError)
       /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
       /p:AzDoPat=$(System.AccessToken)

diff --git a/build-source-tarball.sh b/build-source-tarball.sh
@@ -315,6 +315,17 @@ if [ -e $SCRIPT_ROOT/testing-smoke/smoke-test-packages ]; then
     cp -rf $SCRIPT_ROOT/testing-smoke/smoke-test-packages $TARBALL_ROOT/packages
 fi
 
+echo 'Saving off required arcade prebuilts...'
+mkdir -p $TARBALL_ROOT/packages/arcadeRequired
+while IFS= read -r packagePattern
+do
+    if [[ "$packagePattern" =~ ^# ]]; then
+        continue
+    fi
+    cp -f $TARBALL_ROOT/packages/prebuilt/$packagePattern $TARBALL_ROOT/packages/arcadeRequired/
+done < $SCRIPT_ROOT/support/arcade-required-prebuilts.txt
+
+
 echo 'Removing source-built packages from tarball prebuilts...'
 
 for built_package in $(find $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/blob-feed/packages/ -name '*.nupkg' | tr '[:upper:]' '[:lower:]')

diff --git a/build.sh b/build.sh
@@ -105,6 +105,9 @@ export NUGET_PACKAGES="$scriptroot/packages/restored/"
 set -x
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
+# runtime 3.1.1 required for darc
+"$scriptroot/eng/common/dotnet-install.sh"  -runtime dotnet -version 3.1.1
+
 if [ "$alternateTarget" == "true" ]; then
   "$CLIPATH/dotnet" $SDKPATH/MSBuild.dll "$scriptroot/build.proj" /bl:source-build-test.binlog /flp:v=diag /clp:v=m "$@"
 else

diff --git a/dir.props b/dir.props
@@ -10,11 +10,13 @@
     <!-- true if we have bootstrapped buildtools (usually on an unsupported platform -->
     <_IsBootstrapping Condition="'$(BootstrapBuildToolsDir)' != ''">true</_IsBootstrapping>
 
-    <PortableBuild Condition="'$(OS)' == 'Windows_NT'">true</PortableBuild>
-    <PortableBuild Condition="'$(PortableBuild)' == ''">false</PortableBuild>
-    <UseSystemLibraries Condition="'$(UseSystemLibraries)' == '' AND '$(PortableBuild)' != 'true'">true</UseSystemLibraries>
-    <UseSystemLibunwind Condition="'$(UseSystemLibunwind)' == ''">$(UseSystemLibraries)</UseSystemLibunwind>
     <UseStableVersions Condition="'$(UseStableVersions)' == ''">true</UseStableVersions>
+
+    <!-- new supported portable/nonportable options.  These control whether to build portable runtime
+         or portable SDK.  The PortableBuild flag is only set in runtime-portable.proj and should
+         no longer be passed in.  -->
+    <BuildPortableRuntime Condition="'$(BuildPortableRuntime)' == ''">false</BuildPortableRuntime>
+    <BuildPortableSdk Condition="'$(BuildPortableSdk)' == ''">false</BuildPortableSdk>
   </PropertyGroup>
 
   <PropertyGroup>

diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -1,13 +1,13 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Dependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20151.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20153.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>8ccad075bbb0db445e03eed0a6073d27bdd4f31a</Sha>
+      <Sha>421b20fb860907c45673de791177e3fd61402197</Sha>
     </Dependency>
-    <Dependency Name="Private.SourceBuild.ReferencePackages" Version="1.0.0-beta.20202.1">
+    <Dependency Name="Private.SourceBuild.ReferencePackages" Version="1.0.0-beta.20217.1">
       <Uri>https://github.com/dotnet/source-build-reference-packages</Uri>
-      <Sha>a2a5c06a53a1361a5edf403646aa18f0178dcedb</Sha>
+      <Sha>639aeb4d76c8b1a6226bf7c4edb34fbdae30e6e1</Sha>
     </Dependency>
   </ToolsetDependencies>
   <ProductDependencies>

diff --git a/eng/Versions.props b/eng/Versions.props
@@ -7,7 +7,7 @@
   </PropertyGroup>
   <!-- Production Dependencies -->
   <PropertyGroup>
-    <PrivateSourceBuildReferencePackagesPackageVersion>1.0.0-beta.20202.1</PrivateSourceBuildReferencePackagesPackageVersion>
-    <PrivateSourceBuiltArtifactsPackageVersion>0.1.0-5.0.100-preview.1.20106.2</PrivateSourceBuiltArtifactsPackageVersion>
+    <PrivateSourceBuildReferencePackagesPackageVersion>1.0.0-beta.20217.1</PrivateSourceBuildReferencePackagesPackageVersion>
+    <PrivateSourceBuiltArtifactsPackageVersion>0.1.0-5.0.100-preview.1.20229.1</PrivateSourceBuiltArtifactsPackageVersion>
   </PropertyGroup>
 </Project>
diff --git a/global.json b/global.json
@@ -1,6 +1,6 @@
 {
   "tools": {
-    "dotnet": "3.1.101"
+    "dotnet": "5.0.100-preview.2.20153.3"
   },
   "msbuild-sdks": {
     "Microsoft.Build.CentralPackageVersions": "2.0.1",

diff --git a/patches/arcade/0001-Update-SystemNetHttpPackageVersion.patch b/patches/arcade/0001-Update-SystemNetHttpPackageVersion.patch
@@ -1,4 +1,4 @@
-From 7e4a0c0da30f320bf93e442bb07ec178984d88be Mon Sep 17 00:00:00 2001
+From 89623aca4d2243fab1e7a16eef11d95743bad82a Mon Sep 17 00:00:00 2001
 From: dseefeld <dseefeld@microsoft.com>
 Date: Fri, 17 Jan 2020 10:53:09 +0000
 Subject: [PATCH] Update SystemNetHttpPackageVersion
@@ -8,7 +8,7 @@ Subject: [PATCH] Update SystemNetHttpPackageVersion
  1 file changed, 1 insertion(+), 1 deletion(-)
 
 diff --git a/eng/Versions.props b/eng/Versions.props
-index 5e8edde..a51e53e 100644
+index 34c03ef..70fb1e3 100644
 --- a/eng/Versions.props
 +++ b/eng/Versions.props
 @@ -14,7 +14,7 @@

diff --git a/patches/arcade/0002-Remove-reference-to-Microsoft-CodeAnalysis-in-Packag.patch b/patches/arcade/0002-Remove-reference-to-Microsoft-CodeAnalysis-in-Packag.patch
@@ -1,4 +1,4 @@
-From da96eae5b54e03770ee4a915622d363b15c027b7 Mon Sep 17 00:00:00 2001
+From 934378165868bc05243911836150105cf733b30e Mon Sep 17 00:00:00 2001
 From: dseefeld <dseefeld@microsoft.com>
 Date: Fri, 17 Jan 2020 10:57:21 +0000
 Subject: [PATCH] Remove reference to Microsoft CodeAnalysis in Packaging

diff --git a/patches/arcade/0003-Update-NuGetVersion-to-Reference-Package-version.patch b/patches/arcade/0003-Update-NuGetVersion-to-Reference-Package-version.patch
diff --git a/...mTextJson-version-from-Package-flow.patch → ...mTextJson-version-from-Package-flow.patch b/...mTextJson-version-from-Package-flow.patch → ...mTextJson-version-from-Package-flow.patch
@@ -1,4 +1,4 @@
-From 973f56065a677a2c8054a5a44d1c39af083f0b38 Mon Sep 17 00:00:00 2001
+From a41bf6ff3dfd15ea9b6363b853b8100112cc2b66 Mon Sep 17 00:00:00 2001
 From: dseefeld <dseefeld@microsoft.com>
 Date: Fri, 17 Jan 2020 11:03:36 +0000
 Subject: [PATCH] Update SystemTextJson version from Package flow

diff --git a/patches/arcade/0004-Import-PackageVersions-props-if-exists-and-update.patch b/patches/arcade/0004-Import-PackageVersions-props-if-exists-and-update.patch
diff --git a/...07-Update-versions-of-some-packages.patch → ...04-Update-versions-of-some-packages.patch b/...07-Update-versions-of-some-packages.patch → ...04-Update-versions-of-some-packages.patch
@@ -1,4 +1,4 @@
-From 2456126c6e1e2ec82449f968566ebeffb2921e08 Mon Sep 17 00:00:00 2001
+From 2d63a75def166bf9210c7a032af692b015024df6 Mon Sep 17 00:00:00 2001
 From: dseefeld <dseefeld@microsoft.com>
 Date: Fri, 17 Jan 2020 11:04:50 +0000
 Subject: [PATCH] Update versions of some packages
@@ -8,10 +8,10 @@ Subject: [PATCH] Update versions of some packages
  1 file changed, 2 insertions(+), 2 deletions(-)
 
 diff --git a/eng/Versions.props b/eng/Versions.props
-index e696381..7e1c6b9 100644
+index 70fb1e3..65bc4e3 100644
 --- a/eng/Versions.props
 +++ b/eng/Versions.props
-@@ -44,14 +44,14 @@
+@@ -45,14 +45,14 @@
      <DotNetSleetLibVersion>2.2.143</DotNetSleetLibVersion>
      <SwashbuckleAspNetCoreSwaggerVersion>3.0.0</SwashbuckleAspNetCoreSwaggerVersion>
      <SystemBuffersVersion>4.5.0</SystemBuffersVersion>

diff --git a/...e/0009-Don-t-attempt-to-update-icon.patch → ...e/0005-Don-t-attempt-to-update-icon.patch b/...e/0009-Don-t-attempt-to-update-icon.patch → ...e/0005-Don-t-attempt-to-update-icon.patch
@@ -1,4 +1,4 @@
-From 4c69822eaf6a8e4a6e70ba7546315cbf3101744f Mon Sep 17 00:00:00 2001
+From 72664a1af65826bf5882c68c41710ee3151a4af2 Mon Sep 17 00:00:00 2001
 From: dseefeld <dseefeld@microsoft.com>
 Date: Fri, 17 Jan 2020 11:46:30 +0000
 Subject: [PATCH] Don't attempt to update icon

diff --git a/patches/arcade/0005-Fix-version-of-NewtonsoftJson.patch b/patches/arcade/0005-Fix-version-of-NewtonsoftJson.patch
diff --git a/...or-net472-when-building-from-source.patch → ...or-net472-when-building-from-source.patch b/...or-net472-when-building-from-source.patch → ...or-net472-when-building-from-source.patch
@@ -1,4 +1,4 @@
-From 8df1d92b1091267f8424eb4aa55c4148c1eb4d2b Mon Sep 17 00:00:00 2001
+From 4722f4247c30e8741b9b3d44008f2241844b7357 Mon Sep 17 00:00:00 2001
 From: dseefeld <dseefeld@microsoft.com>
 Date: Fri, 17 Jan 2020 12:28:53 +0000
 Subject: [PATCH] Don't build for net472 when building from source

diff --git a/...eflectionMetadata-from-package-flow.patch → ...eflectionMetadata-from-package-flow.patch b/...eflectionMetadata-from-package-flow.patch → ...eflectionMetadata-from-package-flow.patch
@@ -1,4 +1,4 @@
-From e3c1356f30c5907515bbbc53116338b7f8ba0b0b Mon Sep 17 00:00:00 2001
+From f1563cda0aec56e954350143005dd3082019c9fb Mon Sep 17 00:00:00 2001
 From: dseefeld <dseefeld@microsoft.com>
 Date: Mon, 30 Mar 2020 15:29:50 +0000
 Subject: [PATCH] Use SystemReflectionMetadata from package flow

diff --git a/...clude-Helix-tests-from-source-build.patch → ...clude-Helix-tests-from-source-build.patch b/...clude-Helix-tests-from-source-build.patch → ...clude-Helix-tests-from-source-build.patch
@@ -1,4 +1,4 @@
-From b4bcb878f1792dca75b257e57f098c00a5fb099a Mon Sep 17 00:00:00 2001
+From e821e971b861839cc4b3efceb05b060ae568130d Mon Sep 17 00:00:00 2001
 From: dseefeld <dseefeld@microsoft.com>
 Date: Wed, 1 Apr 2020 16:00:19 +0000
 Subject: [PATCH] Exclude Helix tests from source-build

diff --git a/patches/arcade/0008-Pin-NuGet-versions.patch b/patches/arcade/0008-Pin-NuGet-versions.patch
diff --git a/...d-reference-to-PlatformAbstractions.patch → ...d-reference-to-PlatformAbstractions.patch b/...d-reference-to-PlatformAbstractions.patch → ...d-reference-to-PlatformAbstractions.patch
@@ -1,14 +1,15 @@
-From 75487513ada7c1dafaeb7aff7f287f9c178e4623 Mon Sep 17 00:00:00 2001
+From db4c528bb2335f791f46f76f62f392597e298ef0 Mon Sep 17 00:00:00 2001
 From: dseefeld <dseefeld@microsoft.com>
 Date: Wed, 1 Apr 2020 16:32:18 +0000
 Subject: [PATCH] Add reference to PlatformAbstractions
-Microsoft.DotNet.Build.Tasks.SharedFramework.Sdk.csproj doesn't have a reference 
-to PlatformAbstractions. It builds in the production build because it has a 
-reference to Microsoft.Extensions.DependencyModel, which also brought in 
-PlatformAbstractions. That dependency has been removed with 
-https://github.com/dotnet/core-setup/pull/5218. 
-The tarball build uses a newer version of Microsoft.Extensions.DependencyModel 
-with the dependency removed, so an explicit dependency needs to be added now until 
+
+Microsoft.DotNet.Build.Tasks.SharedFramework.Sdk.csproj doesn't have a reference
+to PlatformAbstractions. It builds in the production build because it has a
+reference to Microsoft.Extensions.DependencyModel, which also brought in
+PlatformAbstractions. That dependency has been removed with
+https://github.com/dotnet/core-setup/pull/5218.
+The tarball build uses a newer version of Microsoft.Extensions.DependencyModel
+with the dependency removed, so an explicit dependency needs to be added now until
 Arcade removes their dependency on PlatormAbstractions.
 ---
  .../Microsoft.DotNet.Build.Tasks.SharedFramework.Sdk.csproj              | 1 +