Skip to content

Commit

Permalink
Start moving over to official templates (#99433)
Browse files Browse the repository at this point in the history
* Start moving over to official templates

* Parameterize runtime-inner-loop

* Use default false for isOfficialBuild in pipeline-with-resources

* Respond to PR comments

* Code review comments

* Use default template args
  • Loading branch information
agocke authored Apr 1, 2024
1 parent dc01e99 commit 4039b45
Show file tree
Hide file tree
Showing 19 changed files with 353 additions and 184 deletions.
30 changes: 17 additions & 13 deletions eng/pipelines/common/global-build-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@ parameters:
enableRichCodeNavigation: false
richCodeNavigationLanguage: 'csharp'
disableComponentGovernance: ''
templatePath: 'templates'

jobs:
- template: /eng/common/templates/job/job.yml
- template: /eng/common/${{ parameters.templatePath }}/job/job.yml
parameters:
${{ if eq(parameters.hostedOs, '') }}:
name: ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }}
Expand Down Expand Up @@ -127,6 +128,7 @@ jobs:

- ${{ each variable in parameters.variables }}:
- ${{ variable }}

steps:
- ${{ if eq(parameters.osGroup, 'windows') }}:
- template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml
Expand Down Expand Up @@ -167,7 +169,7 @@ jobs:
path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles'

- ${{ if eq(parameters.isSourceBuild, true) }}:
- template: /eng/common/templates/steps/source-build.yml
- template: /eng/common/${{ parameters.templatePath }}/steps/source-build.yml
parameters:
platform:
baseOS: ${{ parameters.baseOS }}
Expand Down Expand Up @@ -271,14 +273,16 @@ jobs:
displayName: Collect vslogs on exit
condition: always()

- task: PublishBuildArtifacts@1
displayName: Publish Logs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/'
PublishLocation: Container
${{ if notin(parameters.osGroup, 'browser', 'wasi') }}:
ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
${{ if in(parameters.osGroup, 'browser', 'wasi') }}:
ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
continueOnError: true
condition: always()
- template: /eng/pipelines/common/templates/publish-build-artifacts.yml
parameters:
isOfficialBuild: ${{ parameters.isOfficialBuild }}
displayName: Publish Logs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/'
PublishLocation: Container
${{ if notin(parameters.osGroup, 'browser', 'wasi') }}:
ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
${{ if in(parameters.osGroup, 'browser', 'wasi') }}:
ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
continueOnError: true
condition: always()
232 changes: 121 additions & 111 deletions eng/pipelines/common/templates/pipeline-with-resources.yml
Original file line number Diff line number Diff line change
@@ -1,114 +1,124 @@
parameters:
- name: stages
type: stageList

resources:
containers:
- container: linux_arm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm
env:
ROOTFS_DIR: /crossrootfs/arm

- container: linux_armv6
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10
env:
ROOTFS_DIR: /crossrootfs/armv6

- container: linux_arm64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-biarch-amd64-arm64
env:
ROOTFS_HOST_DIR: /crossrootfs/x64
ROOTFS_DIR: /crossrootfs/arm64

- container: linux_musl_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine
env:
ROOTFS_DIR: /crossrootfs/x64

- container: linux_musl_arm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine
env:
ROOTFS_DIR: /crossrootfs/arm

- container: linux_musl_arm64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine
env:
ROOTFS_DIR: /crossrootfs/arm64

# This container contains all required toolsets to build for Android and for Linux with bionic libc.
- container: linux_bionic
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64

# This container contains all required toolsets to build for Android as well as tooling to build docker images.
- container: android_docker
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker

- container: linux_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64
env:
ROOTFS_DIR: /crossrootfs/x64

- container: linux_x86
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86
env:
ROOTFS_DIR: /crossrootfs/x86

- container: linux_x64_dev_innerloop
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04

# We use a CentOS Stream 8 image here to test building from source on CentOS Stream 8.
- container: SourceBuild_centos_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8

# AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8.
- container: SourceBuild_linux_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build

- container: linux_s390x
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x
env:
ROOTFS_DIR: /crossrootfs/s390x

- container: linux_ppc64le
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le
env:
ROOTFS_DIR: /crossrootfs/ppc64le

- container: linux_riscv64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64
env:
ROOTFS_DIR: /crossrootfs/riscv64

- container: debian-12-gcc13-amd64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64

- container: linux_x64_llvmaot
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8

- container: browser_wasm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
env:
ROOTFS_DIR: /crossrootfs/x64

- container: wasi_wasm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly
env:
ROOTFS_DIR: /crossrootfs/x64

- container: freebsd_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-freebsd-13
env:
ROOTFS_DIR: /crossrootfs/x64

- container: tizen_armel
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-armel-tizen
env:
ROOTFS_DIR: /crossrootfs/armel

- container: debpkg
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg

- container: rpmpkg
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm

stages: ${{ parameters.stages }}
- name: isOfficialBuild
type: boolean
default: false

extends:
template: templateDispatch.yml
parameters:
${{ if parameters.isOfficialBuild }}:
templatePath: template1es.yml
${{ else }}:
templatePath: templatePublic.yml

stages: ${{ parameters.stages }}

containers:
linux_arm:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm
env:
ROOTFS_DIR: /crossrootfs/arm

linux_armv6:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10
env:
ROOTFS_DIR: /crossrootfs/armv6

linux_arm64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-biarch-amd64-arm64
env:
ROOTFS_HOST_DIR: /crossrootfs/x64
ROOTFS_DIR: /crossrootfs/arm64

linux_musl_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine
env:
ROOTFS_DIR: /crossrootfs/x64

linux_musl_arm:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine
env:
ROOTFS_DIR: /crossrootfs/arm

linux_musl_arm64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine
env:
ROOTFS_DIR: /crossrootfs/arm64

# This container contains all required toolsets to build for Android and for Linux with bionic libc.
linux_bionic:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64

# This container contains all required toolsets to build for Android as well as tooling to build docker images.
android_docker:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker

linux_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64
env:
ROOTFS_DIR: /crossrootfs/x64

linux_x86:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86
env:
ROOTFS_DIR: /crossrootfs/x86

linux_x64_dev_innerloop:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04

# We use a CentOS Stream 8 image here to test building from source on CentOS Stream 8.
SourceBuild_centos_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8

# AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8.
SourceBuild_linux_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build

linux_s390x:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x
env:
ROOTFS_DIR: /crossrootfs/s390x

linux_ppc64le:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le
env:
ROOTFS_DIR: /crossrootfs/ppc64le

linux_riscv64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64
env:
ROOTFS_DIR: /crossrootfs/riscv64

debian-12-gcc13-amd64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64

linux_x64_llvmaot:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8

browser_wasm:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
env:
ROOTFS_DIR: /crossrootfs/x64

wasi_wasm:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly
env:
ROOTFS_DIR: /crossrootfs/x64

freebsd_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-freebsd-13
env:
ROOTFS_DIR: /crossrootfs/x64

tizen_armel:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-armel-tizen
env:
ROOTFS_DIR: /crossrootfs/armel

debpkg:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg

rpmpkg:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm
22 changes: 22 additions & 0 deletions eng/pipelines/common/templates/publish-build-artifacts.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
parameters:
- name: isOfficialBuild
type: boolean
- name: displayName
type: string
- name: inputs
type: object
- name: condition
type: string
default: ''

steps:
- ${{ if parameters.isOfficialBuild }}:
- task: 1ES.PublishBuildArtifacts@1
displayName: ${{ parameters.displayName }}
inputs: ${{ parameters.inputs }}
condition: ${{ parameters.condition }}
- ${{ else }}:
- task: PublishBuildArtifacts@1
displayName: ${{ parameters.displayName }}
inputs: ${{ parameters.inputs }}
condition: ${{ parameters.condition }}
17 changes: 17 additions & 0 deletions eng/pipelines/common/templates/publish-pipeline-artifacts.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
parameters:
- name: displayName
type: string
- name: inputs
type: object
- name: isOfficialBuild
type: boolean

steps:
- ${{ if parameters.isOfficialBuild }}:
- task: 1ES.PublishPipelineArtifact@1
displayName: ${{ parameters.displayName }}
inputs: ${{ parameters.inputs }}
- ${{ else }}:
- task: PublishPipelineArtifact@1
displayName: ${{ parameters.displayName }}
inputs: ${{ parameters.inputs }}
18 changes: 10 additions & 8 deletions eng/pipelines/common/templates/runtimes/build-test-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ parameters:
runtimeVariant: ''
dependsOn: []
crossBuild: false
isOfficialBuild: false

### Build managed test components (native components are getting built as part
### of the product build job).
Expand Down Expand Up @@ -137,12 +138,13 @@ jobs:
artifactName: $(microsoftNetSdkIlArtifactName)
displayName: 'Microsoft.NET.Sdk.IL package'


# Publish Logs
- task: PublishPipelineArtifact@1
displayName: Publish Logs
inputs:
targetPath: $(Build.SourcesDirectory)/artifacts/log
artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}'
continueOnError: true
condition: always()
- template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml
parameters:
displayName: Publish Logs
isOfficialBuild: ${{ parameters.isOfficialBuild }}
inputs:
targetPath: $(Build.SourcesDirectory)/artifacts/log
ArtifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}'
continueOnError: true
condition: always()
3 changes: 2 additions & 1 deletion eng/pipelines/common/templates/runtimes/xplat-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@ parameters:
enableMicrobuild: ''
gatherAssetManifests: false
disableComponentGovernance: ''
templatePath: 'templates'

variables: {} ## any extra variables to add to the defaults defined below

jobs:
- template: /eng/common/templates/job/job.yml
- template: /eng/common/${{ parameters.templatePath }}/job/job.yml
parameters:

name: ${{ parameters.name }}
Expand Down
Loading

0 comments on commit 4039b45

Please sign in to comment.