Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CI: split stages in categories (backport #25112) #25260

Closed
wants to merge 4 commits into from
Closed

CI: split stages in categories (backport #25112) #25260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bcdbbd6
CI: split stages in categories (#25112)
v1v Apr 23, 2021
6d921a9
Merge branch '7.12' into mergify/bp/7.12/pr-25112
mergify[bot] Apr 26, 2021
f9ab8d1
Update x-pack/metricbeat/Jenkinsfile.yml
v1v Apr 30, 2021
b9f0095
Merge branch '7.12' into mergify/bp/7.12/pr-25112
mergify[bot] Apr 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 85 additions & 24 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ pipeline {
retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') }
}
}
withMageEnv(version: "${env.GO_VERSION}"){
dir("${BASE_DIR}"){
setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
}
}
}
}
stage('Lint'){
Expand All @@ -77,15 +82,11 @@ pipeline {
withGithubNotify(context: "Lint") {
withBeatsEnv(archive: false, id: "lint") {
dumpVariables()
setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
whenTrue(env.ONLY_DOCS == 'true') {
cmd(label: "make check", script: "make check")
}
whenTrue(env.ONLY_DOCS == 'false') {
cmd(label: "make check-python", script: "make check-python")
cmd(label: "make check-go", script: "make check-go")
cmd(label: "make notice", script: "make notice")
cmd(label: "Check for changes", script: "make check-no-changes")
runLinting()
}
}
}
Expand All @@ -108,28 +109,48 @@ pipeline {
}
}
steps {
deleteDir()
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
dir("${BASE_DIR}"){
script {
def mapParallelTasks = [:]
def content = readYaml(file: 'Jenkinsfile.yml')
if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) {
error 'Pull Request has been configured to be disabled when there is a skip-ci label match'
} else {
content['projects'].each { projectName ->
generateStages(project: projectName, changeset: content['changeset']).each { k,v ->
mapParallelTasks["${k}"] = v
}
}
notifyBuildReason()
parallel(mapParallelTasks)
}
runBuildAndTest(filterStage: 'mandatory')
}
}
stage('Extended') {
options { skipDefaultCheckout() }
when {
// Always when running builds on branches/tags
// On a PR basis, skip if changes are only related to docs.
// Always when forcing the input parameter
anyOf {
not { changeRequest() } // If no PR
allOf { // If PR and no docs changes
expression { return env.ONLY_DOCS == "false" }
changeRequest()
}
expression { return params.runAllStages } // If UI forced
}
}
steps {
runBuildAndTest(filterStage: 'extended')
}
}
stage('Packaging') {
options { skipDefaultCheckout() }
when {
// Always when running builds on branches/tags
// On a PR basis, skip if changes are only related to docs.
// Always when forcing the input parameter
anyOf {
not { changeRequest() } // If no PR
allOf { // If PR and no docs changes
expression { return env.ONLY_DOCS == "false" }
changeRequest()
}
expression { return params.runAllStages } // If UI forced
}
}
steps {
runBuildAndTest(filterStage: 'packaging')
}
}
stage('Packaging-Pipeline') {
agent none
options { skipDefaultCheckout() }
when {
Expand Down Expand Up @@ -166,6 +187,46 @@ VERSION=${env.VERSION}-SNAPSHOT""")
}
}

def runLinting() {
def mapParallelTasks = [:]
def content = readYaml(file: 'Jenkinsfile.yml')
content['projects'].each { projectName ->
generateStages(project: projectName, changeset: content['changeset'], filterStage: 'lint').each { k,v ->
mapParallelTasks["${k}"] = v
}
}
mapParallelTasks['default'] = {
cmd(label: "make check-python", script: "make check-python")
cmd(label: "make check-go", script: "make check-go")
cmd(label: "make notice", script: "make notice")
cmd(label: "Check for changes", script: "make check-no-changes")
}

parallel(mapParallelTasks)
}

def runBuildAndTest(Map args = [:]) {
def filterStage = args.get('filterStage', 'mandatory')
deleteDir()
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
dir("${BASE_DIR}"){
def mapParallelTasks = [:]
def content = readYaml(file: 'Jenkinsfile.yml')
if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) {
error 'Pull Request has been configured to be disabled when there is a skip-ci label match'
} else {
content['projects'].each { projectName ->
generateStages(project: projectName, changeset: content['changeset'], filterStage: filterStage).each { k,v ->
mapParallelTasks["${k}"] = v
}
}
notifyBuildReason()
parallel(mapParallelTasks)
}
}
}


/**
* There are only two supported branches, master and 7.x
*/
Expand All @@ -191,21 +252,21 @@ def getBranchIndice(String compare) {
return 'master'
}


/**
* This method is the one used for running the parallel stages, therefore
* its arguments are passed by the beatsStages step.
*/
def generateStages(Map args = [:]) {
def projectName = args.project
def filterStage = args.get('filterStage', 'all')
def changeset = args.changeset
def mapParallelStages = [:]
def fileName = "${projectName}/Jenkinsfile.yml"
if (fileExists(fileName)) {
def content = readYaml(file: fileName)
// changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation.
if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) {
mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this))
mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this), filterStage: filterStage)
}
} else {
log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml")
Expand Down
34 changes: 24 additions & 10 deletions auditbeat/Jenkinsfile.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@ when:
tags: true ## for all the tags
platform: "immutable && ubuntu-18" ## default label for all the stages
stages:
Lint:
lint:
make: |
make -C auditbeat check;
make -C auditbeat update;
make -C x-pack/auditbeat check;
make -C x-pack/auditbeat update;
make check-no-changes;
stage: lint
arm:
mage: "mage build unitTest"
platforms: ## override default label in this specific stage.
Expand All @@ -33,10 +34,13 @@ stages:
- "armTest"
branches: true ## for all the branches
tags: true ## for all the tags
stage: extended
build:
mage: "mage build test"
stage: mandatory
crosscompile:
make: "make -C auditbeat crosscompile"
stage: mandatory
macos:
mage: "mage build unitTest"
platforms: ## override default label in this specific stage.
Expand All @@ -50,46 +54,56 @@ stages:
- "macosTest"
branches: true ## for all the branches
tags: true ## for all the tags
stage: extended
windows:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2019"
#- "windows-7-32-bit" https://github.com/elastic/beats/issues/19831
#- "windows-2008-r2" https://github.com/elastic/beats/issues/19799
stage: mandatory
windows-2016:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2016"
stage: extended
windows-2012:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2012-r2"
stage: extended
windows-10:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-10"
stage: extended
windows-8:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-8"
stage: extended
#windows-2008: See https://github.com/elastic/beats/issues/19799
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-2008-r2"
# stage: extended
#windows-7: See https://github.com/elastic/beats/issues/19831
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-7"
# when: ## Override the top-level when.
# comments:
# - "/test filebeat for windows-7"
# labels:
# - "windows-7"
# branches: true ## for all the branches
# tags: true ## for all the tags
# stage: extended
#windows-7-32: See https://github.com/elastic/beats/issues/19831
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-7-32-bit"
# stage: extended
packaging-linux:
packaging-linux: "mage package"
e2e:
enabled: false
stage: packaging
packaging-arm:
packaging-arm: "mage package"
e2e:
enabled: false
platforms: ## override default label in this specific stage.
- "arm"
stage: packaging
8 changes: 5 additions & 3 deletions deploy/kubernetes/Jenkinsfile.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@ when:
platform: "immutable && ubuntu-18" ## default label for all the stages
stages:
lint:
make: |
make -C deploy/kubernetes all;
make check-no-changes;
make: |
make -C deploy/kubernetes all;
make check-no-changes;
stage: lint
k8sTest:
k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10"
stage: mandatory
3 changes: 2 additions & 1 deletion dev-tools/Jenkinsfile.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,5 @@ when:
platform: "immutable && ubuntu-18" ## default label for all the stages
stages:
lint:
make: "make -C dev-tools check"
make: "make -C dev-tools check"
stage: lint
20 changes: 13 additions & 7 deletions filebeat/Jenkinsfile.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@ when:
tags: true ## for all the tags
platform: "immutable && ubuntu-18" ## default label for all the stages
stages:
Lint:
lint:
make: |
make -C filebeat check;
make -C filebeat update;
make -C x-pack/filebeat check;
make -C x-pack/filebeat update;
make check-no-changes;
stage: lint
arm:
mage: "mage build unitTest"
platforms: ## override default label in this specific stage.
Expand All @@ -33,9 +34,11 @@ stages:
- "armTest"
branches: true ## for all the branches
tags: true ## for all the tags
stage: extended
build:
mage: "mage build test"
withModule: true ## run the ITs only if the changeset affects a specific module.
stage: mandatory
macos:
mage: "mage build unitTest"
platforms: ## override default label in this specific stage.
Expand All @@ -49,45 +52,48 @@ stages:
- "macosTest"
branches: true ## for all the branches
tags: true ## for all the tags
stage: extended
windows:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2019"
#- "windows-2008-r2" https://github.com/elastic/beats/issues/19795
stage: mandatory
windows-2016:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-2016"
stage: extended
windows-10:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-10"
stage: extended
windows-8:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-8"
stage: extended
#windows-7: See https://github.com/elastic/beats/issues/22317
# mage: "mage build unitTest"
# platforms: ## override default labels in this specific stage.
# - "windows-7"
# when: ## Override the top-level when.
# comments:
# - "/test filebeat for windows-7"
# labels:
# - "windows-7"
# branches: true ## for all the branches
# tags: true ## for all the tags
# stage: packaging
windows-7-32:
mage: "mage build unitTest"
platforms: ## override default labels in this specific stage.
- "windows-7-32-bit"
stage: extended
packaging-linux:
packaging-linux: "mage package"
e2e:
enabled: false
stage: packaging
packaging-arm:
packaging-arm: "mage package"
e2e:
enabled: false
platforms: ## override default label in this specific stage.
- "arm"
stage: packaging
4 changes: 4 additions & 0 deletions generator/Jenkinsfile.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ platform: "immutable && ubuntu-18" ## default label for all the stages
stages:
metricbeat-test:
make: "make -C generator/_templates/metricbeat test test-package"
stage: mandatory
beat-test:
make: "make -C generator/_templates/beat test test-package"
stage: mandatory
macos-metricbeat:
make: "make -C generator/_templates/metricbeat test"
platforms: ## override default label in this specific stage.
Expand All @@ -32,6 +34,7 @@ stages:
- "macosTest"
branches: true ## for all the branches
tags: true ## for all the tags
stage: extended
macos-beat:
make: "make -C generator/_templates/beat test"
platforms: ## override default label in this specific stage.
Expand All @@ -45,3 +48,4 @@ stages:
- "macosTest"
branches: true ## for all the branches
tags: true ## for all the tags
stage: extended
Loading