Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PoC: split mandatory from other stages #24790

Closed
wants to merge 8 commits into from
Closed

PoC: split mandatory from other stages #24790

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2a72a84
PoC: split mandatory from other stages
v1v Mar 26, 2021
b8b5daa
Split for x-pack/winlogbeat
v1v Mar 26, 2021
5540e1d
Run lint per beats earlier
v1v Mar 26, 2021
5d3ae6f
Refactor: add functions
v1v Mar 29, 2021
b917e3a
Rename: markerfile instead definition
v1v Mar 29, 2021
1c161f2
typo with the def
v1v Mar 29, 2021
415e012
Merge branch 'master' into feature/split-pipeline
v1v Apr 7, 2021
0a0faef
Merge remote-tracking branch 'upstream/master' into feature/split-pip…
v1v Apr 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 62 additions & 22 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,7 @@ pipeline {
cmd(label: "make check", script: "make check")
}
whenTrue(env.ONLY_DOCS == 'false') {
cmd(label: "make check-python", script: "make check-python")
cmd(label: "make check-go", script: "make check-go")
cmd(label: "make notice", script: "make notice")
cmd(label: "Check for changes", script: "make check-no-changes")
runLinting()
}
}
}
Expand All @@ -108,26 +105,27 @@ pipeline {
}
}
steps {
deleteDir()
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
dir("${BASE_DIR}"){
script {
def mapParallelTasks = [:]
def content = readYaml(file: 'Jenkinsfile.yml')
if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) {
error 'Pull Request has been configured to be disabled when there is a skip-ci label match'
} else {
content['projects'].each { projectName ->
generateStages(project: projectName, changeset: content['changeset']).each { k,v ->
mapParallelTasks["${k}"] = v
}
}
notifyBuildReason()
parallel(mapParallelTasks)
}
runBuildAndTest(markerfile: 'Jenkinsfile.yml')
}
}
stage('Extended') {
options { skipDefaultCheckout() }
when {
// Always when running builds on branches/tags
// On a PR basis, skip if changes are only related to docs.
// Always when forcing the input parameter
anyOf {
not { changeRequest() } // If no PR
allOf { // If PR and no docs changes
expression { return env.ONLY_DOCS == "false" }
changeRequest()
}
expression { return params.runAllStages } // If UI forced
}
}
steps {
runBuildAndTest(markerfile: 'Jenkinsfile.ext.yml')
}
}
stage('Packaging') {
agent none
Expand Down Expand Up @@ -191,6 +189,47 @@ def getBranchIndice(String compare) {
return 'master'
}

def runLinting() {
def mapParallelTasks = [:]
def content = readYaml(file: 'Jenkinsfile.yml')
content['projects'].each { projectName ->
generateStages(project: projectName, changeset: content['changeset'], markerfile: 'Jenkinsfile.yml').each { k,v ->
if (k.endsWith('lint')) {
mapParallelTasks["${k}"] = v
}
}
}
mapParallelTasks['default'] = {
cmd(label: "make check-python", script: "make check-python")
cmd(label: "make check-go", script: "make check-go")
cmd(label: "make notice", script: "make notice")
cmd(label: "Check for changes", script: "make check-no-changes")
}

parallel(mapParallelTasks)
}

def runBuildAndTest(Map args = [:]) {
deleteDir()
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
dir("${BASE_DIR}"){
def mapParallelTasks = [:]
def content = readYaml(file: 'Jenkinsfile.yml')
if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) {
error 'Pull Request has been configured to be disabled when there is a skip-ci label match'
} else {
content['projects'].each { projectName ->
generateStages(project: projectName, changeset: content['changeset'], markerfile: args.markerfile).each { k,v ->
if (!k.endsWith('lint')) {
mapParallelTasks["${k}"] = v
}
}
}
notifyBuildReason()
parallel(mapParallelTasks)
}
}
}

/**
* This method is the one used for running the parallel stages, therefore
Expand All @@ -199,8 +238,9 @@ def getBranchIndice(String compare) {
def generateStages(Map args = [:]) {
def projectName = args.project
def changeset = args.changeset
def markerfile = args.get('markerfile', 'Jenkinsfile.yml')
def mapParallelStages = [:]
def fileName = "${projectName}/Jenkinsfile.yml"
def fileName = "${projectName}/${markerfile}"
if (fileExists(fileName)) {
def content = readYaml(file: fileName)
// changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation.
Expand Down
66 changes: 66 additions & 0 deletions auditbeat/Jenkinsfile.ext.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
when:
branches: true ## for all the branches
changeset: ## when PR contains any of those entries in the changeset
- "^auditbeat/.*"
- "@ci" ## special token regarding the changeset for the ci
- "@oss" ## special token regarding the changeset for the oss
comments: ## when PR comment contains any of those entries
- "/test ext auditbeat"
labels: ## when PR labels matches any of those entries
- "auditbeat"
parameters: ## when parameter was selected in the UI.
- "auditbeat"
tags: true ## for all the tags
platform: "immutable && ubuntu-18" ## default label for all the stages
stages:
macos:
mage: "mage build unitTest"
platforms: ## override default label in this specific stage.
- "macosx&&x86_64"
when: ## Override the top-level when.
comments:
- "/test auditbeat for macos"
labels:
- "macOS"
parameters:
- "macosTest"
branches: true ## for all the branches
tags: true ## for all the tags
windows-2016:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2016"
windows-2012:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2012-r2"
windows-10:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-10"
windows-8:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-8"
#windows-2008: See https://github.com/elastic/beats/issues/19799
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-2008-r2"
#windows-7-32: See https://github.com/elastic/beats/issues/19831
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-7-32-bit"
#windows-7: See https://github.com/elastic/beats/issues/19831
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-7"
packaging-linux:
packaging-linux: "mage package"
e2e:
enabled: false
packaging-arm:
packaging-arm: "mage package"
e2e:
enabled: false
platforms: ## override default label in this specific stage.
- "arm"
67 changes: 3 additions & 64 deletions auditbeat/Jenkinsfile.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,83 +13,22 @@ when:
tags: true ## for all the tags
platform: "immutable && ubuntu-18" ## default label for all the stages
stages:
Lint:
lint:
make: |
make -C auditbeat check;
make -C auditbeat update;
make -C x-pack/auditbeat check;
make -C x-pack/auditbeat update;
make check-no-changes;
arm:
mage: "mage build unitTest"
platforms: ## override default label in this specific stage.
- "arm"
when: ## Override the top-level when.
comments:
- "/test auditbeat for arm"
labels:
- "arm"
parameters:
- "armTest"
branches: true ## for all the branches
tags: true ## for all the tags
build:
mage: "mage build test"
crosscompile:
make: "make -C auditbeat crosscompile"
macos:
arm:
mage: "mage build unitTest"
platforms: ## override default label in this specific stage.
- "macosx&&x86_64"
when: ## Override the top-level when.
comments:
- "/test auditbeat for macos"
labels:
- "macOS"
parameters:
- "macosTest"
branches: true ## for all the branches
tags: true ## for all the tags
- "arm"
windows:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2019"
#- "windows-7-32-bit" https://github.com/elastic/beats/issues/19831
#- "windows-2008-r2" https://github.com/elastic/beats/issues/19799
windows-2016:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2016"
windows-2012:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2012-r2"
windows-10:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-10"
windows-8:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-8"
#windows-7: See https://github.com/elastic/beats/issues/19831
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-7"
# when: ## Override the top-level when.
# comments:
# - "/test filebeat for windows-7"
# labels:
# - "windows-7"
# branches: true ## for all the branches
# tags: true ## for all the tags
packaging-linux:
packaging-linux: "mage package"
e2e:
enabled: false
packaging-arm:
packaging-arm: "mage package"
e2e:
enabled: false
platforms: ## override default label in this specific stage.
- "arm"
69 changes: 69 additions & 0 deletions filebeat/Jenkinsfile.ext.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
when:
branches: true ## for all the branches
changeset: ## when PR contains any of those entries in the changeset
- "^filebeat/.*"
- "@ci" ## special token regarding the changeset for the ci
- "@oss" ## special token regarding the changeset for the oss
comments: ## when PR comment contains any of those entries
- "/test ext filebeat"
labels: ## when PR labels matches any of those entries
- "filebeat"
parameters: ## when parameter was selected in the UI.
- "filebeat"
tags: true ## for all the tags
platform: "immutable && ubuntu-18" ## default label for all the stages
stages:
macos:
mage: "mage build unitTest"
platforms: ## override default label in this specific stage.
- "macosx&&x86_64"
when: ## Override the top-level when.
comments:
- "/test filebeat for macos"
labels:
- "macOS"
parameters:
- "macosTest"
branches: true ## for all the branches
tags: true ## for all the tags
windows-2016:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2016"
#windows-2008: https://github.com/elastic/beats/issues/19795
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-2008-r2"
windows-10:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-10"
windows-8:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-8"
#windows-7: See https://github.com/elastic/beats/issues/22317
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-7"
# when: ## Override the top-level when.
# comments:
# - "/test filebeat for windows-7"
# labels:
# - "windows-7"
# branches: true ## for all the branches
# tags: true ## for all the tags
windows-7-32:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-7-32-bit"
packaging-linux:
packaging-linux: "mage package"
e2e:
enabled: false
packaging-arm:
packaging-arm: "mage package"
e2e:
enabled: false
platforms: ## override default label in this specific stage.
- "arm"
Loading