From 74cb414cf47a8653ae15ead49461ef7cf7eeb2d4 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Thu, 26 Mar 2020 15:40:45 -0700 Subject: [PATCH 01/28] Add support for building S.P.CoreLib in a separate job --- .../coreclr/templates/build-corelib-job.yml | 163 ++++++++++++++++++ eng/pipelines/runtime.yml | 38 ++++ 2 files changed, 201 insertions(+) create mode 100644 eng/pipelines/coreclr/templates/build-corelib-job.yml diff --git a/eng/pipelines/coreclr/templates/build-corelib-job.yml b/eng/pipelines/coreclr/templates/build-corelib-job.yml new file mode 100644 index 0000000000000..a7d10fda90857 --- /dev/null +++ b/eng/pipelines/coreclr/templates/build-corelib-job.yml @@ -0,0 +1,163 @@ +parameters: + archType: '' + buildConfig: '' + compilerName: 'clang' + condition: true + container: '' + crossrootfsDir: '' + isOfficialBuild: false + osGroup: '' + osSubgroup: '' + platform: '' + pool: '' + signBinaries: false + stagedBuild: false + testGroup: '' + timeoutInMinutes: '' + variables: {} + +### Product build +jobs: +- template: xplat-pipeline-job.yml + parameters: + buildConfig: ${{ parameters.buildConfig }} + _BuildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + testGroup: ${{ parameters.testGroup }} + helixType: 'build/product/' + enableMicrobuild: true + stagedBuild: ${{ parameters.stagedBuild }} + pool: ${{ parameters.pool }} + condition: ${{ parameters.condition }} + + # Compute job name from template parameters + ${{ if eq(parameters.compilerName, 'gcc') }}: + name: ${{ format('coreclr_{0}_corelib_build_{1}{1}_{3}_{4}', parameters.compilerName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('CoreCLR GCC Product Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if eq(parameters.compilerName, 'clang') }}: + name: ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('CoreCLR Product Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + + # Run all steps in the container. + # Note that the containers are defined in platform-matrix.yml + container: ${{ parameters.container }} + + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + crossrootfsDir: ${{ parameters.crossrootfsDir }} + + gatherAssetManifests: true + variables: + - name: osGroup + value: ${{ parameters.osGroup }} + - name: osSubgroup + value: ${{ parameters.osSubgroup }} + - name: stripSymbolsArg + value: '' + # Strip symbols only on the release build + - ${{ if eq(parameters.buildConfig, 'Release') }}: + - name: stripSymbolsArg + value: '-stripsymbols' + - name: compilerArg + value: '' + - name: publishLogsArtifactPrefix + value: 'BuildLogs_CoreCLR' + - ${{ if eq(parameters.compilerName, 'gcc') }}: + - name: compilerArg + value: '-gcc' + - name: publishLogsArtifactPrefix + value: 'BuildLogs_CoreCLR_GCC' + # workaround for gcc directory not in PATH + - name: CLR_CC + value: /opt/rh/devtoolset-7/root/usr/bin/gcc + - name: CLR_CXX + value: /opt/rh/devtoolset-7/root/usr/bin/g++ + - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.compilerName, 'clang')) }}: + - name: compilerArg + value: '-clang9' + # Our FreeBSD doesn't yet detect available clang versions, so pass it explicitly. + - ${{ if eq(parameters.osGroup, 'FreeBSD') }}: + - name: compilerArg + value: '-clang6.0' + # Building for x64 MUSL happens on Alpine Linux and we need to use the stable version available there + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64')) }}: + - name: compilerArg + value: '' + # AppleClang has different version scheme, so we let complier introspection pick up the available clang from PATH + - ${{ if eq(parameters.osGroup, 'OSX') }}: + - name: compilerArg + value: '' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: + # Variables used to publish packages to blob feed + - name: dotnetfeedUrl + value: https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json + - name: dotnetfeedPAT + value: $(dotnetfeed-storage-access-key-1) + # Variables used by arcade to gather asset manifests + - name: _DotNetPublishToBlobFeed + value: true + - name: officialBuildIdArg + value: '' + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - name: officialBuildIdArg + value: '/p:OfficialBuildId=$(Build.BuildNumber)' + - name: enforcePgoArg + value: '' + # The EnforcePGO script is only supported on Windows and is not supported on arm or arm64. + - ${{ if and(eq(parameters.buildConfig, 'Release'), and(eq(parameters.osGroup, 'Windows_NT'), not(or(eq(parameters.archType, 'arm64'), eq(parameters.archType, 'arm'))))) }}: + - name: enforcePgoArg + value: '-enforcepgo' + + - ${{ parameters.variables }} + + steps: + + # Install native dependencies + # Linux builds use docker images with dependencies preinstalled, + # and FreeBSD builds use a build agent with dependencies + # preinstalled, so we only need this step for OSX and Windows. + - ${{ if eq(parameters.osGroup, 'OSX') }}: + - script: sh $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) + displayName: Install native dependencies + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + # Necessary to install python + - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force + displayName: Install native dependencies + + # # Install internal tools on official builds + # # Since our internal tools are behind an authenticated feed, + # # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. + # # We can't do this from within the build, so we need to do this as a separate step. + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/pipelines/common/restore-internal-tools.yml + parameters: + installDotnet: true + + # Build Private CoreLib + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subsetCategory coreclr -subset corelib $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci + displayName: Build System.Private.CoreLib only + + # Get key vault secrets for publishing + - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - task: AzureKeyVault@1 + inputs: + azureSubscription: 'DotNet-Engineering-Services_KeyVault' + KeyVaultName: EngKeyVault + SecretsFilter: 'dotnetfeed-storage-access-key-1,microsoft-symbol-server-pat,symweb-symbol-server-pat' + + # Save packages using the prepare-signed-artifacts format. + - ${{ if eq(parameters.isOfficialBuild, true) }}: + - template: /eng/pipelines/common/upload-unsigned-artifacts-step.yml + parameters: + name: ${{ parameters.platform }} + + # Publish Logs + - task: PublishPipelineArtifact@1 + displayName: Publish Logs + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: '$(publishLogsArtifactPrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + continueOnError: true + condition: always() diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 8f909825a4311..8921d1d8bd270 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -118,6 +118,44 @@ jobs: - eng/pipelines/mono/* - eng/pipelines/libraries/* +# +# Build CoreLib checked +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: checked + platforms: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm + - Windows_NT_arm64 + jobParameters: + testGroup: innerloop + +# +# Build CoreLib release +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: release + platforms: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm + - Windows_NT_arm64 + jobParameters: + testGroup: innerloop + # # Build CoreCLR checked # Only when CoreCLR is changed From def5d70c8da20c69693e07384a709731bb0cc8e1 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Thu, 26 Mar 2020 20:31:19 -0700 Subject: [PATCH 02/28] PR Comments. Removing compilerName parameter and renaming display names --- .../coreclr/templates/build-corelib-job.yml | 39 ++----------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-corelib-job.yml b/eng/pipelines/coreclr/templates/build-corelib-job.yml index a7d10fda90857..8b8807aa1fd22 100644 --- a/eng/pipelines/coreclr/templates/build-corelib-job.yml +++ b/eng/pipelines/coreclr/templates/build-corelib-job.yml @@ -1,7 +1,6 @@ parameters: archType: '' buildConfig: '' - compilerName: 'clang' condition: true container: '' crossrootfsDir: '' @@ -32,14 +31,9 @@ jobs: pool: ${{ parameters.pool }} condition: ${{ parameters.condition }} - # Compute job name from template parameters - ${{ if eq(parameters.compilerName, 'gcc') }}: - name: ${{ format('coreclr_{0}_corelib_build_{1}{1}_{3}_{4}', parameters.compilerName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('CoreCLR GCC Product Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if eq(parameters.compilerName, 'clang') }}: - name: ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('CoreCLR Product Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - + name: ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('CoreLib Product Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml container: ${{ parameters.container }} @@ -63,24 +57,7 @@ jobs: - name: compilerArg value: '' - name: publishLogsArtifactPrefix - value: 'BuildLogs_CoreCLR' - - ${{ if eq(parameters.compilerName, 'gcc') }}: - - name: compilerArg - value: '-gcc' - - name: publishLogsArtifactPrefix - value: 'BuildLogs_CoreCLR_GCC' - # workaround for gcc directory not in PATH - - name: CLR_CC - value: /opt/rh/devtoolset-7/root/usr/bin/gcc - - name: CLR_CXX - value: /opt/rh/devtoolset-7/root/usr/bin/g++ - - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.compilerName, 'clang')) }}: - - name: compilerArg - value: '-clang9' - # Our FreeBSD doesn't yet detect available clang versions, so pass it explicitly. - - ${{ if eq(parameters.osGroup, 'FreeBSD') }}: - - name: compilerArg - value: '-clang6.0' + value: 'BuildLogs_CoreLib' # Building for x64 MUSL happens on Alpine Linux and we need to use the stable version available there - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64')) }}: - name: compilerArg @@ -139,14 +116,6 @@ jobs: - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subsetCategory coreclr -subset corelib $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci displayName: Build System.Private.CoreLib only - # Get key vault secrets for publishing - - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - task: AzureKeyVault@1 - inputs: - azureSubscription: 'DotNet-Engineering-Services_KeyVault' - KeyVaultName: EngKeyVault - SecretsFilter: 'dotnetfeed-storage-access-key-1,microsoft-symbol-server-pat,symweb-symbol-server-pat' - # Save packages using the prepare-signed-artifacts format. - ${{ if eq(parameters.isOfficialBuild, true) }}: - template: /eng/pipelines/common/upload-unsigned-artifacts-step.yml From fc99faadcfc1783286e0924336daef904191791c Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Thu, 26 Mar 2020 20:36:25 -0700 Subject: [PATCH 03/28] fix deletion typo --- eng/pipelines/coreclr/templates/build-corelib-job.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-corelib-job.yml b/eng/pipelines/coreclr/templates/build-corelib-job.yml index 8b8807aa1fd22..0ec3c2fd378c5 100644 --- a/eng/pipelines/coreclr/templates/build-corelib-job.yml +++ b/eng/pipelines/coreclr/templates/build-corelib-job.yml @@ -58,14 +58,6 @@ jobs: value: '' - name: publishLogsArtifactPrefix value: 'BuildLogs_CoreLib' - # Building for x64 MUSL happens on Alpine Linux and we need to use the stable version available there - - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64')) }}: - - name: compilerArg - value: '' - # AppleClang has different version scheme, so we let complier introspection pick up the available clang from PATH - - ${{ if eq(parameters.osGroup, 'OSX') }}: - - name: compilerArg - value: '' - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: # Variables used to publish packages to blob feed - name: dotnetfeedUrl From de8f01243e7e80bbe1235830a58375cc7aa65b57 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Fri, 27 Mar 2020 10:51:38 -0700 Subject: [PATCH 04/28] Adding step to upload output directory. Also some cleanup --- .../coreclr/templates/build-corelib-job.yml | 27 +++++++++---------- .../coreclr/templates/xplat-pipeline-job.yml | 3 +++ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-corelib-job.yml b/eng/pipelines/coreclr/templates/build-corelib-job.yml index 0ec3c2fd378c5..dedd4029588a5 100644 --- a/eng/pipelines/coreclr/templates/build-corelib-job.yml +++ b/eng/pipelines/coreclr/templates/build-corelib-job.yml @@ -58,26 +58,11 @@ jobs: value: '' - name: publishLogsArtifactPrefix value: 'BuildLogs_CoreLib' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: - # Variables used to publish packages to blob feed - - name: dotnetfeedUrl - value: https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json - - name: dotnetfeedPAT - value: $(dotnetfeed-storage-access-key-1) - # Variables used by arcade to gather asset manifests - - name: _DotNetPublishToBlobFeed - value: true - name: officialBuildIdArg value: '' - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - name: officialBuildIdArg value: '/p:OfficialBuildId=$(Build.BuildNumber)' - - name: enforcePgoArg - value: '' - # The EnforcePGO script is only supported on Windows and is not supported on arm or arm64. - - ${{ if and(eq(parameters.buildConfig, 'Release'), and(eq(parameters.osGroup, 'Windows_NT'), not(or(eq(parameters.archType, 'arm64'), eq(parameters.archType, 'arm'))))) }}: - - name: enforcePgoArg - value: '-enforcepgo' - ${{ parameters.variables }} @@ -108,6 +93,18 @@ jobs: - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subsetCategory coreclr -subset corelib $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci displayName: Build System.Private.CoreLib only + + # Publish corelib output directory for consumption by libraries. + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(buildProductRootFolderPath) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: $(corelibProductArtifactName) + displayName: 'corelib build' + # Save packages using the prepare-signed-artifacts format. - ${{ if eq(parameters.isOfficialBuild, true) }}: - template: /eng/pipelines/common/upload-unsigned-artifacts-step.yml diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index ae418ed1533e7..aad6f7bf69fa4 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -75,6 +75,9 @@ jobs: - name: buildProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' + - name: corelibProductArtifactName + value: 'CoreLib_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + - name: managedTestArtifactName value: 'CoreCLRManagedTestArtifacts_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_$(archType)_$(buildConfig)' From 34a19e8a96d69d603966cc417f3edeb449e47c3c Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Fri, 27 Mar 2020 11:43:38 -0700 Subject: [PATCH 05/28] update libraries job to depend on corelib instead of product --- eng/pipelines/libraries/base-job.yml | 2 +- eng/pipelines/libraries/build-job.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index e24d0d3328bbc..454988437e44d 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -96,7 +96,7 @@ jobs: - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' - - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' + - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 4f95464e683df..81393e5e400a1 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -49,9 +49,9 @@ jobs: ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: dependsOn: - ${{ if ne(parameters.osGroup, 'WebAssembly') }}: - - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} + - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: - - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} + - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} variables: - _skipTestRestoreArg: /p:SkipTestRestore=true From eb1ee34888e3553bdf7b728e7ab712d3418f7fb8 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Fri, 27 Mar 2020 14:31:25 -0700 Subject: [PATCH 06/28] reverting libraries dependency change Keeping the download of corelib artifacts to check if just corelib is sufficient to build libraires. --- eng/pipelines/libraries/build-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 81393e5e400a1..4f95464e683df 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -49,9 +49,9 @@ jobs: ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: dependsOn: - ${{ if ne(parameters.osGroup, 'WebAssembly') }}: - - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} + - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: - - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} + - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} variables: - _skipTestRestoreArg: /p:SkipTestRestore=true From 478b77fbe07da3f848e02b8591713b1046e1b9f4 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Fri, 27 Mar 2020 17:11:07 -0700 Subject: [PATCH 07/28] Add OSX and musl_arm64 corelib builds + use full product for mono --- eng/pipelines/libraries/base-job.yml | 7 ++++++- eng/pipelines/runtime.yml | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 454988437e44d..86288a016edfb 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -96,7 +96,12 @@ jobs: - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' - - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' + # Use corelib dependencies for coreclr + - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: + - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' + # Use full product dependencies for mono + - ${{ if eq(parameters.runtimeFlavor, 'mono') }}: + - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 8921d1d8bd270..604ba57f7563d 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -145,12 +145,14 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml buildConfig: release platforms: + - OSX_x64 - Linux_x64 - Linux_arm - Linux_arm64 - Linux_musl_x64 - - Windows_NT_x86 + - Linux_musl_arm64 - Windows_NT_x64 + - Windows_NT_x86 - Windows_NT_arm - Windows_NT_arm64 jobParameters: From a6a2a3b88e24977753f5316adc634c48d993ce0f Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Fri, 27 Mar 2020 21:50:53 -0700 Subject: [PATCH 08/28] Another attempt to fix mono iOS case --- eng/pipelines/libraries/base-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 86288a016edfb..f9b918d6adf02 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -8,7 +8,7 @@ parameters: isOfficialBuild: false isOfficialAllConfigurations: false liveRuntimeBuildConfig: '' - runtimeFlavor: 'coreclr' + runtimeFlavor: '' timeoutInMinutes: 150 condition: true container: '' @@ -95,7 +95,7 @@ jobs: - _runtimeArtifactsPathArg: '' - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' + - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/$(runtimeFlavorName)' # Use corelib dependencies for coreclr - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' From fbfb35e43f9a342a450323415640bb713e964be0 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Fri, 27 Mar 2020 22:01:33 -0700 Subject: [PATCH 09/28] propagating runtimeFlavor to libraries base-job.yml --- eng/pipelines/libraries/build-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 4f95464e683df..6fe6a39f8e4c7 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -37,6 +37,7 @@ jobs: isOfficialBuild: ${{ parameters.isOfficialBuild }} isOfficialAllConfigurations: ${{ parameters.isOfficialAllConfigurations }} liveRuntimeBuildConfig: ${{ parameters.liveRuntimeBuildConfig }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} preBuildSteps: ${{ parameters.preBuildSteps }} container: ${{ parameters.container }} From 46efc4d4f3e0a703feb7d1cc588a9ee9fe8dfc3b Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Fri, 27 Mar 2020 22:28:28 -0700 Subject: [PATCH 10/28] updating DependsOn for libraries job to use corelib --- eng/pipelines/libraries/build-job.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 6fe6a39f8e4c7..8ca2cb9eb63d4 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -50,7 +50,11 @@ jobs: ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: dependsOn: - ${{ if ne(parameters.osGroup, 'WebAssembly') }}: - - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} + # Use corelib dependencies for coreclr + - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: + - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} + - ${{ if eq(parameters.runtimeFlavor, 'mono') }}: + - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} From 57b5d81508aa1d9b175c723ea7a8cee537c72eb0 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Sat, 28 Mar 2020 08:28:36 -0700 Subject: [PATCH 11/28] Make LIbraries Test Build also depend on Product --- eng/pipelines/libraries/build-test-job.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml index 15a0540876150..b821c45cdda4f 100644 --- a/eng/pipelines/libraries/build-test-job.yml +++ b/eng/pipelines/libraries/build-test-job.yml @@ -6,6 +6,7 @@ parameters: framework: '' isOfficialBuild: false liveRuntimeBuildConfig: '' + runtimeFlavor: 'coreclr' timeoutInMinutes: 150 container: '' publishTestArtifacs: true @@ -38,6 +39,9 @@ jobs: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if in(parameters.framework, 'allConfigurations', 'net472') }}: - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + # Libraries Test also depends on Product, now that the libraries build only depends on corelib + - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: + - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} variables: - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} From 6436fb673700c00812fabca0f41401fcc5210910 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Sat, 28 Mar 2020 09:21:58 -0700 Subject: [PATCH 12/28] download product dependency when testScope is not null --- eng/pipelines/libraries/base-job.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index c653463c2ac21..9d5b9b53c28f3 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -95,11 +95,11 @@ jobs: - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/$(runtimeFlavorName)' - # Use corelib dependencies for coreclr - - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: + # Download corelib dependencies for coreclr libraries, if not test + - ${{ if and(ne(parameters.runtimeFlavor, 'mono'), eq(parameters.testScope, '')) }}: - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - # Use full product dependencies for mono - - ${{ if eq(parameters.runtimeFlavor, 'mono') }}: + # Download full product dependencies for mono or test + - ${{ if or(eq(parameters.runtimeFlavor, 'mono'), ne(parameters.testScope, '')) }}: - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} From 0683d82edd074ed60b4620ad8561fb6f66821b7f Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Sat, 28 Mar 2020 10:54:16 -0700 Subject: [PATCH 13/28] reverting a change which is not required Hoping this fixes the test build. --- eng/pipelines/libraries/base-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 9d5b9b53c28f3..26ca9277d1e8b 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -8,7 +8,7 @@ parameters: isOfficialBuild: false isOfficialAllConfigurations: false liveRuntimeBuildConfig: '' - runtimeFlavor: '' + runtimeFlavor: 'coreclr' timeoutInMinutes: 150 condition: true container: '' @@ -94,8 +94,8 @@ jobs: - _runtimeArtifactsPathArg: '' - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/$(runtimeFlavorName)' - # Download corelib dependencies for coreclr libraries, if not test + - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' + # Download corelib dependencies for coreclr libraries and not test - ${{ if and(ne(parameters.runtimeFlavor, 'mono'), eq(parameters.testScope, '')) }}: - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' # Download full product dependencies for mono or test From f21a514ea86869de4ede5624e3bfe2e8d55f5e9b Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Sun, 29 Mar 2020 22:10:26 -0700 Subject: [PATCH 14/28] Adding runtime.depproj to libraries restore, and pretest.proj to tests.proj pretest is required to generate the Microsoft.NETCore.App.deps files for CoreRun to work correctly. --- src/libraries/restore/dirs.proj | 1 + src/libraries/tests.proj | 1 + 2 files changed, 2 insertions(+) diff --git a/src/libraries/restore/dirs.proj b/src/libraries/restore/dirs.proj index d006b65c8ff7e..ecdfa828672dc 100644 --- a/src/libraries/restore/dirs.proj +++ b/src/libraries/restore/dirs.proj @@ -8,6 +8,7 @@ + diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index d47a26ac972b8..06a08c9f16d2d 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -14,6 +14,7 @@ + From 19e670112118116bc9d4fef3e94ad94a6ad39c38 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Sun, 29 Mar 2020 22:54:30 -0700 Subject: [PATCH 15/28] Fixing run_test_job to download product artifact --- eng/pipelines/libraries/run-test-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 2c6d1be9a6665..02b43448d8744 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -31,6 +31,7 @@ jobs: container: '' # we just send to helix, no need to use a container. condition: ${{ parameters.condition }} pool: ${{ parameters.pool }} + testScope: ${{ parameters.testScope }} ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: displayName: ${{ format('Test Run {0} {1}', parameters.liveRuntimeBuildConfig, parameters.runtimeFlavor) }} name: ${{ format('test_run_{0}_{1}', parameters.liveRuntimeBuildConfig, parameters.runtimeFlavor) }} From 6b73ed578b93ebda673e5b20e4a600a6802b82da Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Mon, 30 Mar 2020 01:02:10 -0700 Subject: [PATCH 16/28] run pretest.proj on test runs as well. --- eng/pipelines/libraries/run-test-job.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 02b43448d8744..83f03dd74978f 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -82,6 +82,16 @@ jobs: /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideCoreclrFromLiveDrop.binlog displayName: Override $(runtimeFlavorName) from live drop + - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: + - script: $(_msbuildCommand) src/libraries/pretest.proj + $(_runtimeArtifactsPathArg) + $(_runtimeOSArg) + /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} + /p:ArchGroup=${{ parameters.archType }} + /p:Configuration=${{ parameters.buildConfig }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideCoreclrFromLiveDrop_pretest.binlog + displayName: Override $(runtimeFlavorName) from live drop PreTest + - template: /eng/pipelines/libraries/helix.yml parameters: runtimeFlavor: ${{ parameters.runtimeFlavor }} From bb09589b5a702aeab75d9471792b951fcd46bbe6 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Mon, 30 Mar 2020 07:40:27 -0700 Subject: [PATCH 17/28] Add libraries -restore to test run since pretest needs installer.tasks.dll --- eng/pipelines/libraries/run-test-job.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 83f03dd74978f..50d2840fa1fc5 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -72,6 +72,9 @@ jobs: artifactName: $(librariesTestsArtifactName) artifactFileName: $(librariesTestsArtifactName)$(archiveExtension) + - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) + displayName: Restore + - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - script: $(_msbuildCommand) src/libraries/restore/runtime/runtime.depproj $(_runtimeArtifactsPathArg) From 2ec1f5a66f84f52f6998ee96ba57352c226b7c17 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Mon, 30 Mar 2020 13:50:20 -0700 Subject: [PATCH 18/28] Updating pretest to use TargetArchitecture A recent change caused this break :) --- eng/pipelines/libraries/run-test-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 50d2840fa1fc5..db79b4d47fd8a 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -90,7 +90,7 @@ jobs: $(_runtimeArtifactsPathArg) $(_runtimeOSArg) /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - /p:ArchGroup=${{ parameters.archType }} + /p:TargetArchitecture=${{ parameters.archType }} /p:Configuration=${{ parameters.buildConfig }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideCoreclrFromLiveDrop_pretest.binlog displayName: Override $(runtimeFlavorName) from live drop PreTest From e6d48826b13a0528323840b5bf951b40390baf5a Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Mon, 30 Mar 2020 21:41:55 -0700 Subject: [PATCH 19/28] Moving runtime restore from \restore\dirs.proj Moved runtime restore + pretest.proj to teh tests.proj build. --- src/libraries/restore/dirs.proj | 1 - src/libraries/tests.proj | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libraries/restore/dirs.proj b/src/libraries/restore/dirs.proj index ecdfa828672dc..d006b65c8ff7e 100644 --- a/src/libraries/restore/dirs.proj +++ b/src/libraries/restore/dirs.proj @@ -8,7 +8,6 @@ - diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index 06a08c9f16d2d..3b83970388d76 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -14,7 +14,6 @@ - @@ -44,6 +43,11 @@ + + Date: Mon, 30 Mar 2020 23:32:30 -0700 Subject: [PATCH 20/28] fix -allConfigurations build --- src/libraries/tests.proj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index 3b83970388d76..8b4fe9410ba10 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -46,6 +46,7 @@ Date: Tue, 31 Mar 2020 07:45:36 -0700 Subject: [PATCH 21/28] Adding corelib job to coreclr=and=Libraries. --- .../common/build-coreclr-and-libraries-job.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/eng/pipelines/common/build-coreclr-and-libraries-job.yml b/eng/pipelines/common/build-coreclr-and-libraries-job.yml index ddf5d8103cac7..7ea5e7af112cf 100644 --- a/eng/pipelines/common/build-coreclr-and-libraries-job.yml +++ b/eng/pipelines/common/build-coreclr-and-libraries-job.yml @@ -13,6 +13,21 @@ parameters: pool: '' jobs: +- template: /eng/pipelines/coreclr/templates/build-corelib-job.yml + parameters: + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + container: ${{ parameters.container }} + testGroup: ${{ parameters.testGroup }} + crossrootfsDir: ${{ parameters.crossrootfsDir }} + timeoutInminutes: ${{ parameters.timeoutInMinutes }} + signBinaries: ${{ parameters.signBinaries }} + stagedBuild: ${{ parameters.stagedBuild }} + variables: ${{ parameters.variables }} + pool: ${{ parameters.pool }} + - template: /eng/pipelines/coreclr/templates/build-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} From fec325d7df0c9514de7ee31dcb8bb889ee22e947 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Tue, 31 Mar 2020 10:28:14 -0700 Subject: [PATCH 22/28] Add corelib to other pipelines. --- eng/pipelines/coreclr/ci.yml | 46 +++++++++++++++++++++++++++ eng/pipelines/libraries/outerloop.yml | 23 ++++++++++++++ eng/pipelines/runtime-official.yml | 21 ++++++++++++ 3 files changed, 90 insertions(+) diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index 0249dc33eba23..8bf0391de2f14 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -31,6 +31,52 @@ jobs: # - template: /eng/pipelines/common/checkout-job.yml +# +# Debug Corelib builds +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: debug + platforms: + - Linux_arm + - Linux_arm64 + - Linux_musl_arm64 + - Linux_musl_x64 + - Linux_x64 + - OSX_x64 + - Windows_NT_arm + - Windows_NT_arm64 + jobParameters: + testGroup: outerloop + +# +# Checked Corelib builds +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: checked + platformGroup: all + jobParameters: + testGroup: outerloop + +# +# Release Corelib builds +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: release + platforms: + - Linux_arm + - Linux_musl_arm64 + - Linux_x64 + - OSX_x64 + - Windows_NT_x86 + jobParameters: + testGroup: outerloop + # # Debug builds # diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml index dd7fb83ae9a73..eb3b2e9493ddf 100644 --- a/eng/pipelines/libraries/outerloop.yml +++ b/eng/pipelines/libraries/outerloop.yml @@ -14,6 +14,29 @@ variables: jobs: - template: /eng/pipelines/common/checkout-job.yml + # + # CoreLib Build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: release + platforms: + - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: + - Windows_NT_x64 + - Windows_NT_x86 + - ${{ if eq(variables['includeLinuxOuterloop'], true) }}: + - Linux_x64 + - Linux_musl_x64 + - ${{ if eq(variables['isFullMatrix'], true) }}: + - Linux_arm + - Linux_arm64 + - Linux_musl_arm64 + - ${{ if eq(variables['includeOsxOuterloop'], true) }}: + - OSX_x64 + jobParameters: + testGroup: innerloop + # # CoreCLR Build # diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index a49a4b1bbda73..45c5740583268 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -39,6 +39,27 @@ stages: # - template: /eng/pipelines/common/checkout-job.yml + # + # Build CoreLib + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: release + platforms: + - OSX_x64 + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Linux_musl_arm64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm + - Windows_NT_arm64 + jobParameters: + isOfficialBuild: ${{ variables.isOfficialBuild }} + # # Build CoreCLR # From dfef8c304fb09497d70e000e31345832a250d6d5 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Tue, 31 Mar 2020 14:20:16 -0700 Subject: [PATCH 23/28] libraries test should depend on full product This fix is similar to artifact download logic done in build-base.yml --- eng/pipelines/libraries/build-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 8ca2cb9eb63d4..a06e927c97262 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -51,9 +51,9 @@ jobs: dependsOn: - ${{ if ne(parameters.osGroup, 'WebAssembly') }}: # Use corelib dependencies for coreclr - - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: + - ${{ if and(ne(parameters.runtimeFlavor, 'mono'), eq(parameters.testScope, '')) }}: - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - - ${{ if eq(parameters.runtimeFlavor, 'mono') }}: + - ${{ if or(eq(parameters.runtimeFlavor, 'mono'), ne(parameters.testScope, '')) }}: - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} From 2e4d00a891ffd53080e7ce51abeb0fe6f4039955 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Tue, 31 Mar 2020 16:07:12 -0700 Subject: [PATCH 24/28] remove upload unsigned artifacts step from official build --- eng/pipelines/coreclr/templates/build-corelib-job.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-corelib-job.yml b/eng/pipelines/coreclr/templates/build-corelib-job.yml index dedd4029588a5..8f34004c69c38 100644 --- a/eng/pipelines/coreclr/templates/build-corelib-job.yml +++ b/eng/pipelines/coreclr/templates/build-corelib-job.yml @@ -105,12 +105,6 @@ jobs: artifactName: $(corelibProductArtifactName) displayName: 'corelib build' - # Save packages using the prepare-signed-artifacts format. - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/upload-unsigned-artifacts-step.yml - parameters: - name: ${{ parameters.platform }} - # Publish Logs - task: PublishPipelineArtifact@1 displayName: Publish Logs From 56bf4c030c484248218767557761974cee8bd17d Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Wed, 1 Apr 2020 15:58:14 -0700 Subject: [PATCH 25/28] PR Comments --- .../coreclr/templates/build-corelib-job.yml | 12 +++--------- eng/pipelines/libraries/base-job.yml | 4 ++-- eng/pipelines/libraries/build-job.yml | 6 +++--- eng/pipelines/libraries/run-test-job.yml | 10 +++------- src/libraries/tests.proj | 9 +++++++-- 5 files changed, 18 insertions(+), 23 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-corelib-job.yml b/eng/pipelines/coreclr/templates/build-corelib-job.yml index 8f34004c69c38..bfb91cb4d9095 100644 --- a/eng/pipelines/coreclr/templates/build-corelib-job.yml +++ b/eng/pipelines/coreclr/templates/build-corelib-job.yml @@ -32,7 +32,7 @@ jobs: condition: ${{ parameters.condition }} name: ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('CoreLib Product Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('CoreCLR CoreLib Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -48,12 +48,6 @@ jobs: value: ${{ parameters.osGroup }} - name: osSubgroup value: ${{ parameters.osSubgroup }} - - name: stripSymbolsArg - value: '' - # Strip symbols only on the release build - - ${{ if eq(parameters.buildConfig, 'Release') }}: - - name: stripSymbolsArg - value: '-stripsymbols' - name: compilerArg value: '' - name: publishLogsArtifactPrefix @@ -91,7 +85,7 @@ jobs: # Build Private CoreLib - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subsetCategory coreclr -subset corelib $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci - displayName: Build System.Private.CoreLib only + displayName: Build System.Private.CoreLib # Publish corelib output directory for consumption by libraries. @@ -103,7 +97,7 @@ jobs: tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) artifactName: $(corelibProductArtifactName) - displayName: 'corelib build' + displayName: Upload System.Private.CoreLib # Publish Logs - task: PublishPipelineArtifact@1 diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 26ca9277d1e8b..3f9f5f6ed8b14 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -96,10 +96,10 @@ jobs: - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' # Download corelib dependencies for coreclr libraries and not test - - ${{ if and(ne(parameters.runtimeFlavor, 'mono'), eq(parameters.testScope, '')) }}: + - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}: - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' # Download full product dependencies for mono or test - - ${{ if or(eq(parameters.runtimeFlavor, 'mono'), ne(parameters.testScope, '')) }}: + - ${{ if or(neq(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index a06e927c97262..28c4738cd4861 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -50,10 +50,10 @@ jobs: ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: dependsOn: - ${{ if ne(parameters.osGroup, 'WebAssembly') }}: - # Use corelib dependencies for coreclr - - ${{ if and(ne(parameters.runtimeFlavor, 'mono'), eq(parameters.testScope, '')) }}: + # Use corelib dependencies for coreclr and non test builds + - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}: - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - - ${{ if or(eq(parameters.runtimeFlavor, 'mono'), ne(parameters.testScope, '')) }}: + - ${{ if or(neq(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index f4de128a6dc07..17f4dae14d395 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -71,9 +71,6 @@ jobs: unpackFolder: $(Build.SourcesDirectory)/artifacts artifactName: $(librariesTestsArtifactName) artifactFileName: $(librariesTestsArtifactName)$(archiveExtension) - - - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) - displayName: Restore - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - script: $(_msbuildCommand) src/libraries/restore/runtime/runtime.depproj @@ -82,17 +79,16 @@ jobs: /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /p:TargetArchitecture=${{ parameters.archType }} /p:Configuration=${{ parameters.buildConfig }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideCoreclrFromLiveDrop.binlog + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideRuntimeFromLiveDrop.binlog displayName: Override $(runtimeFlavorName) from live drop - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - script: $(_msbuildCommand) src/libraries/pretest.proj + - script: $(_buildScript) -project src/libraries/pretest.proj $(_runtimeArtifactsPathArg) $(_runtimeOSArg) /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /p:TargetArchitecture=${{ parameters.archType }} /p:Configuration=${{ parameters.buildConfig }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideCoreclrFromLiveDrop_pretest.binlog + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/pretest.binlog displayName: Override $(runtimeFlavorName) from live drop PreTest - template: /eng/pipelines/libraries/helix.yml diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index 8b4fe9410ba10..975b8c7763d46 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -41,12 +41,17 @@ + + + + + Date: Wed, 1 Apr 2020 16:05:05 -0700 Subject: [PATCH 26/28] typo fix :( --- eng/pipelines/libraries/base-job.yml | 2 +- eng/pipelines/libraries/build-job.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 3f9f5f6ed8b14..deee63825eff1 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -99,7 +99,7 @@ jobs: - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}: - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' # Download full product dependencies for mono or test - - ${{ if or(neq(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: + - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 28c4738cd4861..14057daa4ff90 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -53,7 +53,7 @@ jobs: # Use corelib dependencies for coreclr and non test builds - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}: - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - - ${{ if or(neq(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: + - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} From e76db3a650930729cd2652b8939fd731223cd483 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Wed, 1 Apr 2020 18:02:00 -0700 Subject: [PATCH 27/28] Fix run-test-job pretest build --- eng/pipelines/libraries/run-test-job.yml | 7 ++++--- src/libraries/pretest.proj | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 17f4dae14d395..6035ec9730c7b 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -71,7 +71,7 @@ jobs: unpackFolder: $(Build.SourcesDirectory)/artifacts artifactName: $(librariesTestsArtifactName) artifactFileName: $(librariesTestsArtifactName)$(archiveExtension) - + - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - script: $(_msbuildCommand) src/libraries/restore/runtime/runtime.depproj $(_runtimeArtifactsPathArg) @@ -82,14 +82,15 @@ jobs: /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideRuntimeFromLiveDrop.binlog displayName: Override $(runtimeFlavorName) from live drop - - script: $(_buildScript) -project src/libraries/pretest.proj + - script: $(_buildScript) -projects $(Build.SourcesDirectory)/src/libraries/pretest.proj $(_runtimeArtifactsPathArg) $(_runtimeOSArg) /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /p:TargetArchitecture=${{ parameters.archType }} /p:Configuration=${{ parameters.buildConfig }} + /p:SkipBuildProjects=true /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/pretest.binlog - displayName: Override $(runtimeFlavorName) from live drop PreTest + displayName: Build PreTest to Generate Deps Json File - template: /eng/pipelines/libraries/helix.yml parameters: diff --git a/src/libraries/pretest.proj b/src/libraries/pretest.proj index 317bf140a39af..10cd240779387 100644 --- a/src/libraries/pretest.proj +++ b/src/libraries/pretest.proj @@ -78,7 +78,8 @@ Projects="@(Project)" Properties="TargetFramework=$(BuildTargetFramework);%(Project.AdditionalProperties)" BuildInParallel="true" - ContinueOnError="ErrorAndStop" /> + ContinueOnError="ErrorAndStop" + Condition="'$(SkipBuildProjects)' != 'true'"/> From e3c15b0b1be2e388ee9f0776f29c8d0762979b23 Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Wed, 1 Apr 2020 23:36:10 -0700 Subject: [PATCH 28/28] removed some unused corelib configs. --- eng/pipelines/coreclr/ci.yml | 35 ----------------------------------- eng/pipelines/runtime.yml | 19 ------------------- 2 files changed, 54 deletions(-) diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index 8bf0391de2f14..0848c4cc3ea8c 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -31,25 +31,6 @@ jobs: # - template: /eng/pipelines/common/checkout-job.yml -# -# Debug Corelib builds -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml - buildConfig: debug - platforms: - - Linux_arm - - Linux_arm64 - - Linux_musl_arm64 - - Linux_musl_x64 - - Linux_x64 - - OSX_x64 - - Windows_NT_arm - - Windows_NT_arm64 - jobParameters: - testGroup: outerloop - # # Checked Corelib builds # @@ -61,22 +42,6 @@ jobs: jobParameters: testGroup: outerloop -# -# Release Corelib builds -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml - buildConfig: release - platforms: - - Linux_arm - - Linux_musl_arm64 - - Linux_x64 - - OSX_x64 - - Windows_NT_x86 - jobParameters: - testGroup: outerloop - # # Debug builds # diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 604ba57f7563d..5edbdb1d88a35 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -118,25 +118,6 @@ jobs: - eng/pipelines/mono/* - eng/pipelines/libraries/* -# -# Build CoreLib checked -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml - buildConfig: checked - platforms: - - Linux_x64 - - Linux_arm - - Linux_arm64 - - Linux_musl_x64 - - Windows_NT_x86 - - Windows_NT_x64 - - Windows_NT_arm - - Windows_NT_arm64 - jobParameters: - testGroup: innerloop - # # Build CoreLib release #