From f4c570b06ff5bf8a4fd079da23df0cf10ef0318b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 23 Apr 2021 09:09:06 +0100 Subject: [PATCH] CI: split stages in categories (#25112) --- Jenkinsfile | 109 +++++++++++++++++++++------ auditbeat/Jenkinsfile.yml | 34 ++++++--- deploy/kubernetes/Jenkinsfile.yml | 8 +- dev-tools/Jenkinsfile.yml | 3 +- filebeat/Jenkinsfile.yml | 20 +++-- generator/Jenkinsfile.yml | 4 + heartbeat/Jenkinsfile.yml | 16 +++- journalbeat/Jenkinsfile.yml | 7 +- libbeat/Jenkinsfile.yml | 6 +- metricbeat/Jenkinsfile.yml | 17 ++++- packetbeat/Jenkinsfile.yml | 16 +++- winlogbeat/Jenkinsfile.yml | 18 ++++- x-pack/auditbeat/Jenkinsfile.yml | 16 +++- x-pack/dockerlogbeat/Jenkinsfile.yml | 6 +- x-pack/elastic-agent/Jenkinsfile.yml | 23 ++++-- x-pack/filebeat/Jenkinsfile.yml | 25 +++--- x-pack/functionbeat/Jenkinsfile.yml | 14 +++- x-pack/heartbeat/Jenkinsfile.yml | 15 +++- x-pack/libbeat/Jenkinsfile.yml | 5 +- x-pack/metricbeat/Jenkinsfile.yml | 16 +++- x-pack/osquerybeat/Jenkinsfile.yml | 13 +++- x-pack/packetbeat/Jenkinsfile.yml | 16 +++- x-pack/winlogbeat/Jenkinsfile.yml | 12 ++- 23 files changed, 341 insertions(+), 78 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 30479827d91..087f6bd98e1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,6 +65,11 @@ pipeline { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } } } + withMageEnv(version: "${env.GO_VERSION}"){ + dir("${BASE_DIR}"){ + setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) + } + } } } stage('Lint'){ @@ -77,15 +82,11 @@ pipeline { withGithubNotify(context: "Lint") { withBeatsEnv(archive: false, id: "lint") { dumpVariables() - setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) whenTrue(env.ONLY_DOCS == 'true') { cmd(label: "make check", script: "make check") } whenTrue(env.ONLY_DOCS == 'false') { - cmd(label: "make check-python", script: "make check-python") - cmd(label: "make check-go", script: "make check-go") - cmd(label: "make notice", script: "make notice") - cmd(label: "Check for changes", script: "make check-no-changes") + runLinting() } } } @@ -108,28 +109,48 @@ pipeline { } } steps { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - script { - def mapParallelTasks = [:] - def content = readYaml(file: 'Jenkinsfile.yml') - if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) { - error 'Pull Request has been configured to be disabled when there is a skip-ci label match' - } else { - content['projects'].each { projectName -> - generateStages(project: projectName, changeset: content['changeset']).each { k,v -> - mapParallelTasks["${k}"] = v - } - } - notifyBuildReason() - parallel(mapParallelTasks) - } + runBuildAndTest(filterStage: 'mandatory') + } + } + stage('Extended') { + options { skipDefaultCheckout() } + when { + // Always when running builds on branches/tags + // On a PR basis, skip if changes are only related to docs. + // Always when forcing the input parameter + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() } + expression { return params.runAllStages } // If UI forced } } + steps { + runBuildAndTest(filterStage: 'extended') + } } stage('Packaging') { + options { skipDefaultCheckout() } + when { + // Always when running builds on branches/tags + // On a PR basis, skip if changes are only related to docs. + // Always when forcing the input parameter + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() + } + expression { return params.runAllStages } // If UI forced + } + } + steps { + runBuildAndTest(filterStage: 'packaging') + } + } + stage('Packaging-Pipeline') { agent none options { skipDefaultCheckout() } when { @@ -166,6 +187,46 @@ VERSION=${env.VERSION}-SNAPSHOT""") } } +def runLinting() { + def mapParallelTasks = [:] + def content = readYaml(file: 'Jenkinsfile.yml') + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset'], filterStage: 'lint').each { k,v -> + mapParallelTasks["${k}"] = v + } + } + mapParallelTasks['default'] = { + cmd(label: "make check-python", script: "make check-python") + cmd(label: "make check-go", script: "make check-go") + cmd(label: "make notice", script: "make notice") + cmd(label: "Check for changes", script: "make check-no-changes") + } + + parallel(mapParallelTasks) +} + +def runBuildAndTest(Map args = [:]) { + def filterStage = args.get('filterStage', 'mandatory') + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + dir("${BASE_DIR}"){ + def mapParallelTasks = [:] + def content = readYaml(file: 'Jenkinsfile.yml') + if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) { + error 'Pull Request has been configured to be disabled when there is a skip-ci label match' + } else { + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset'], filterStage: filterStage).each { k,v -> + mapParallelTasks["${k}"] = v + } + } + notifyBuildReason() + parallel(mapParallelTasks) + } + } +} + + /** * There are only two supported branches, master and 7.x */ @@ -191,13 +252,13 @@ def getBranchIndice(String compare) { return 'master' } - /** * This method is the one used for running the parallel stages, therefore * its arguments are passed by the beatsStages step. */ def generateStages(Map args = [:]) { def projectName = args.project + def filterStage = args.get('filterStage', 'all') def changeset = args.changeset def mapParallelStages = [:] def fileName = "${projectName}/Jenkinsfile.yml" @@ -205,7 +266,7 @@ def generateStages(Map args = [:]) { def content = readYaml(file: fileName) // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation. if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) { - mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this)) + mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this), filterStage: filterStage) } } else { log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index f2059b80601..876696ded99 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C auditbeat check; make -C auditbeat update; make -C x-pack/auditbeat check; make -C x-pack/auditbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,10 +34,13 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory crosscompile: make: "make -C auditbeat crosscompile" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -50,46 +54,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - #- "windows-7-32-bit" https://github.com/elastic/beats/issues/19831 - #- "windows-2008-r2" https://github.com/elastic/beats/issues/19799 + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended + #windows-2008: See https://github.com/elastic/beats/issues/19799 + # mage: "mage build unitTest" + # platforms: ## override default labels in this specific stage. + # - "windows-2008-r2" + # stage: extended #windows-7: See https://github.com/elastic/beats/issues/19831 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7" - # when: ## Override the top-level when. - # comments: - # - "/test filebeat for windows-7" - # labels: - # - "windows-7" - # branches: true ## for all the branches - # tags: true ## for all the tags + # stage: extended + #windows-7-32: See https://github.com/elastic/beats/issues/19831 + # mage: "mage build unitTest" + # platforms: ## override default labels in this specific stage. + # - "windows-7-32-bit" + # stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml index 50c95bee667..107e15107c3 100644 --- a/deploy/kubernetes/Jenkinsfile.yml +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -13,8 +13,10 @@ when: platform: "immutable && ubuntu-18" ## default label for all the stages stages: lint: - make: | - make -C deploy/kubernetes all; - make check-no-changes; + make: | + make -C deploy/kubernetes all; + make check-no-changes; + stage: lint k8sTest: k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" + stage: mandatory diff --git a/dev-tools/Jenkinsfile.yml b/dev-tools/Jenkinsfile.yml index e1970328515..98c2f4db210 100644 --- a/dev-tools/Jenkinsfile.yml +++ b/dev-tools/Jenkinsfile.yml @@ -13,4 +13,5 @@ when: platform: "immutable && ubuntu-18" ## default label for all the stages stages: lint: - make: "make -C dev-tools check" + make: "make -C dev-tools check" + stage: lint diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index ac8899504ac..121abfbb6ba 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C filebeat check; make -C filebeat update; make -C x-pack/filebeat check; make -C x-pack/filebeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,9 +34,11 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,45 +52,48 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" #- "windows-2008-r2" https://github.com/elastic/beats/issues/19795 + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended #windows-7: See https://github.com/elastic/beats/issues/22317 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7" # when: ## Override the top-level when. - # comments: - # - "/test filebeat for windows-7" - # labels: - # - "windows-7" - # branches: true ## for all the branches - # tags: true ## for all the tags + # stage: packaging windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml index f3210ee482a..9cc3f948a73 100644 --- a/generator/Jenkinsfile.yml +++ b/generator/Jenkinsfile.yml @@ -17,8 +17,10 @@ platform: "immutable && ubuntu-18" ## default label for all the stages stages: metricbeat-test: make: "make -C generator/_templates/metricbeat test test-package" + stage: mandatory beat-test: make: "make -C generator/_templates/beat test test-package" + stage: mandatory macos-metricbeat: make: "make -C generator/_templates/metricbeat test" platforms: ## override default label in this specific stage. @@ -32,6 +34,7 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended macos-beat: make: "make -C generator/_templates/beat test" platforms: ## override default label in this specific stage. @@ -45,3 +48,4 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 5f4324820fe..b605fea3e8f 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C heartbeat check; make -C heartbeat update; make -C x-pack/heartbeat check; make -C x-pack/heartbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,8 +34,10 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -48,45 +51,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index a91d99dd115..17444786286 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -13,11 +13,12 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C journalbeat check; make -C journalbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -31,15 +32,19 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended unitTest: mage: "mage build unitTest" + stage: mandatory packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 42ca46b2afb..1e9f7724028 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -12,13 +12,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C libbeat check; make -C libbeat update; make -C x-pack/libbeat check; make -C x-pack/libbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -32,7 +33,10 @@ stages: - "armTest" build: mage: "mage build test" + stage: mandatory crosscompile: make: "make -C libbeat crosscompile" + stage: mandatory stress-tests: make: "make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests" + stage: mandatory diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 7d7b4a6227e..57f18ff81d2 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -13,23 +13,28 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C metricbeat check; make -C metricbeat update; make -C x-pack/metricbeat check; make -C x-pack/metricbeat update; make check-no-changes; + stage: lint unitTest: mage: "mage build unitTest" + stage: mandatory goIntegTest: mage: "mage goIntegTest" withModule: true + stage: mandatory pythonIntegTest: mage: "mage pythonIntegTest" withModule: true + stage: mandatory crosscompile: make: "make -C metricbeat crosscompile" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -47,41 +52,51 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false ## e2e is enabled only for x-pack beats + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 659b3878364..86cf53094b9 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -13,7 +13,7 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C packetbeat check; make -C packetbeat update; @@ -21,6 +21,7 @@ stages: cd x-pack/packetbeat; mage check; mage update; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -34,8 +35,10 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,45 +52,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index a66ffcf1c73..d90f7500946 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -13,45 +13,59 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C winlogbeat check; make -C winlogbeat update; make -C x-pack/winlogbeat check; make -C x-pack/winlogbeat update; make check-no-changes; + stage: lint crosscompile: make: "make -C winlogbeat crosscompile" + stage: mandatory windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - - "windows-2008-r2" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended + windows-2008: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 663cd82b161..73ebf4b28ec 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/auditbeat check; make -C x-pack/auditbeat update; make -C auditbeat check; make -C auditbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,9 +34,11 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage update build test" withModule: true ## run the ITs only if the changeset affects a specific module. + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,45 +52,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml index d31c4e32b5f..c608d5ed54f 100644 --- a/x-pack/dockerlogbeat/Jenkinsfile.yml +++ b/x-pack/dockerlogbeat/Jenkinsfile.yml @@ -13,21 +13,25 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/dockerlogbeat check; make -C x-pack/dockerlogbeat update; make check-no-changes; + stage: lint build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. + stage: mandatory packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index d85a96c71f6..7b45fe7ae7a 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -13,11 +13,12 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/elastic-agent check; make -C x-pack/elastic-agent update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -31,8 +32,10 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -46,52 +49,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended #windows-7-32: See https://github.com/elastic/beats/issues/22316 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7-32-bit" - # when: ## Override the top-level when. - # comments: - # - "/test x-pack/elastic-agent for windows-7-32" - # labels: - # - "windows-7-32" - # branches: true ## for all the branches - # tags: true ## for all the tags + # stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index cc13fe11bc3..1246581eb0d 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/filebeat check; make -C x-pack/filebeat update; make -C filebeat check; make -C filebeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,9 +34,11 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,52 +52,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended #windows-7-32: See https://github.com/elastic/beats/issues/22315 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7-32-bit" - # when: ## Override the top-level when. - # comments: - # - "/test x-pack/filebeat for windows-7-32" - # labels: - # - "windows-7-32" - # branches: true ## for all the branches - # tags: true ## for all the tags + # stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - - "arm" \ No newline at end of file + - "arm" + stage: packaging \ No newline at end of file diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 1dede6243e5..8128d9d716f 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -13,11 +13,12 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/functionbeat check; make -C x-pack/functionbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -31,6 +32,7 @@ stages: - "armTest" build: mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -44,39 +46,49 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging diff --git a/x-pack/heartbeat/Jenkinsfile.yml b/x-pack/heartbeat/Jenkinsfile.yml index ef2429d3dec..bdf8b8f9817 100644 --- a/x-pack/heartbeat/Jenkinsfile.yml +++ b/x-pack/heartbeat/Jenkinsfile.yml @@ -13,15 +13,17 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/heartbeat check; make -C x-pack/heartbeat update; make -C heartbeat check; make -C heartbeat update; make check-no-changes; + stage: lint build: mage: "mage build test" + stage: mandatory macos: mage: "mage build test" platforms: ## override default label in this specific stage. @@ -35,6 +37,7 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended # TODO: there are windows test failures already reported # https://github.com/elastic/beats/issues/23957 and https://github.com/elastic/beats/issues/23958 # waiting for being fixed. @@ -42,41 +45,51 @@ stages: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-2019" +# stage: mandatory # windows-2016: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-2016" +# stage: extended # windows-2012: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-2012-r2" +# stage: extended # windows-10: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-10" +# stage: extended # windows-2008: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-2008-r2" +# stage: extended # windows-8: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-8" +# stage: extended # windows-7: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-7" +# stage: extended # windows-7-32: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-7-32-bit" +# stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 267cf1be9ac..5d295d73904 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -13,11 +13,12 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/libbeat check; make -C x-pack/libbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -31,5 +32,7 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 497fbc4b5eb..e82fd9baef6 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -13,15 +13,17 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/metricbeat check; make -C x-pack/metricbeat update; make -C metricbeat check; make -C metricbeat update; make check-no-changes; + stage: lint unitTest: mage: "mage build unitTest" + stage: mandatory cloud: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. @@ -34,6 +36,7 @@ stages: - "/test x-pack/metricbeat for aws cloud" labels: - "aws" + stage: extended macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -47,47 +50,58 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended # windows-7-32 builder has been disabled due to https://github.com/elastic/beats/issues/24337 #windows-7-32: # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7-32-bit" + # stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false entrypoint: 'metricbeat-test.sh' + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/osquerybeat/Jenkinsfile.yml b/x-pack/osquerybeat/Jenkinsfile.yml index d3c591e0d7e..7ed2ac5e088 100644 --- a/x-pack/osquerybeat/Jenkinsfile.yml +++ b/x-pack/osquerybeat/Jenkinsfile.yml @@ -13,13 +13,15 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/osquerybeat check; make -C x-pack/osquerybeat update; make check-no-changes; + stage: lint build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,35 +35,44 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index e35dfa34879..40227865b5f 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -13,7 +13,7 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: mage: | mage check; mage update; @@ -21,6 +21,7 @@ stages: make -C packetbeat check; make -C packetbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -34,8 +35,10 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,46 +52,57 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" withModule: true platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index 8b03fed80a3..fe37a3f40a9 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -13,7 +13,7 @@ when: tags: true ## for all the tags platform: "windows-2019" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/winlogbeat check; make -C x-pack/winlogbeat update; @@ -22,42 +22,52 @@ stages: make check-no-changes; platforms: ## override default labels in this specific stage. - "immutable && ubuntu-18" + stage: lint build: mage: "mage build unitTest" withModule: true platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false platforms: ## override default labels in this specific stage. - "immutable && ubuntu-18" + stage: packaging