From 4c6715609d5015bce29aa5075af05bc729a0b8c5 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 15 Apr 2021 15:05:33 +0100 Subject: [PATCH 1/9] CI: four sequential stages(lint, mandatory, extended, packaging) --- Jenkinsfile | 104 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 22 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 30479827d91..353bea2a476 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -82,10 +82,7 @@ pipeline { cmd(label: "make check", script: "make check") } whenTrue(env.ONLY_DOCS == 'false') { - cmd(label: "make check-python", script: "make check-python") - cmd(label: "make check-go", script: "make check-go") - cmd(label: "make notice", script: "make notice") - cmd(label: "Check for changes", script: "make check-no-changes") + runLinting() } } } @@ -108,28 +105,48 @@ pipeline { } } steps { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - script { - def mapParallelTasks = [:] - def content = readYaml(file: 'Jenkinsfile.yml') - if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) { - error 'Pull Request has been configured to be disabled when there is a skip-ci label match' - } else { - content['projects'].each { projectName -> - generateStages(project: projectName, changeset: content['changeset']).each { k,v -> - mapParallelTasks["${k}"] = v - } - } - notifyBuildReason() - parallel(mapParallelTasks) - } + runBuildAndTest(filter: 'mandatory') + } + } + stage('Extended') { + options { skipDefaultCheckout() } + when { + // Always when running builds on branches/tags + // On a PR basis, skip if changes are only related to docs. + // Always when forcing the input parameter + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() } + expression { return params.runAllStages } // If UI forced } } + steps { + runBuildAndTest(filter: 'extended') + } } stage('Packaging') { + options { skipDefaultCheckout() } + when { + // Always when running builds on branches/tags + // On a PR basis, skip if changes are only related to docs. + // Always when forcing the input parameter + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() + } + expression { return params.runAllStages } // If UI forced + } + } + steps { + runBuildAndTest(filter: 'packaging') + } + } + stage('Packaging-Pipeline') { agent none options { skipDefaultCheckout() } when { @@ -166,6 +183,50 @@ VERSION=${env.VERSION}-SNAPSHOT""") } } +def runLinting() { + def mapParallelTasks = [:] + def content = readYaml(file: 'Jenkinsfile.yml') + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset']).each { k,v -> + if (v?.stage?.equals('lint')) { + mapParallelTasks["${k}"] = v + } + } + } + mapParallelTasks['default'] = { + cmd(label: "make check-python", script: "make check-python") + cmd(label: "make check-go", script: "make check-go") + cmd(label: "make notice", script: "make notice") + cmd(label: "Check for changes", script: "make check-no-changes") + } + + parallel(mapParallelTasks) +} + +def runBuildAndTest(Map args = [:]) { + def filter = args.get('filter', 'mandatory') + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + dir("${BASE_DIR}"){ + def mapParallelTasks = [:] + def content = readYaml(file: 'Jenkinsfile.yml') + if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) { + error 'Pull Request has been configured to be disabled when there is a skip-ci label match' + } else { + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset'], filter: args.filter).each { k,v -> + if (v?.stage?.equals(filter)) { + mapParallelTasks["${k}"] = v + } + } + } + notifyBuildReason() + parallel(mapParallelTasks) + } + } +} + + /** * There are only two supported branches, master and 7.x */ @@ -191,7 +252,6 @@ def getBranchIndice(String compare) { return 'master' } - /** * This method is the one used for running the parallel stages, therefore * its arguments are passed by the beatsStages step. From 08f32ca4dabe7a400c3f8a2a07aabbb09f372334 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 15 Apr 2021 15:11:13 +0100 Subject: [PATCH 2/9] Split stages in four categories --- auditbeat/Jenkinsfile.yml | 34 ++++++++++++++++++++-------- deploy/kubernetes/Jenkinsfile.yml | 8 ++++--- dev-tools/Jenkinsfile.yml | 3 ++- filebeat/Jenkinsfile.yml | 20 ++++++++++------ generator/Jenkinsfile.yml | 4 ++++ heartbeat/Jenkinsfile.yml | 16 ++++++++++++- journalbeat/Jenkinsfile.yml | 7 +++++- libbeat/Jenkinsfile.yml | 6 ++++- metricbeat/Jenkinsfile.yml | 17 +++++++++++++- packetbeat/Jenkinsfile.yml | 16 ++++++++++++- winlogbeat/Jenkinsfile.yml | 18 +++++++++++++-- x-pack/auditbeat/Jenkinsfile.yml | 16 ++++++++++++- x-pack/dockerlogbeat/Jenkinsfile.yml | 6 ++++- x-pack/elastic-agent/Jenkinsfile.yml | 23 ++++++++++++------- x-pack/filebeat/Jenkinsfile.yml | 25 ++++++++++++-------- x-pack/functionbeat/Jenkinsfile.yml | 14 +++++++++++- x-pack/heartbeat/Jenkinsfile.yml | 15 +++++++++++- x-pack/libbeat/Jenkinsfile.yml | 5 +++- x-pack/metricbeat/Jenkinsfile.yml | 16 ++++++++++++- x-pack/osquerybeat/Jenkinsfile.yml | 13 ++++++++++- x-pack/packetbeat/Jenkinsfile.yml | 16 ++++++++++++- x-pack/winlogbeat/Jenkinsfile.yml | 12 +++++++++- 22 files changed, 256 insertions(+), 54 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index f2059b80601..876696ded99 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C auditbeat check; make -C auditbeat update; make -C x-pack/auditbeat check; make -C x-pack/auditbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,10 +34,13 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory crosscompile: make: "make -C auditbeat crosscompile" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -50,46 +54,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - #- "windows-7-32-bit" https://github.com/elastic/beats/issues/19831 - #- "windows-2008-r2" https://github.com/elastic/beats/issues/19799 + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended + #windows-2008: See https://github.com/elastic/beats/issues/19799 + # mage: "mage build unitTest" + # platforms: ## override default labels in this specific stage. + # - "windows-2008-r2" + # stage: extended #windows-7: See https://github.com/elastic/beats/issues/19831 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7" - # when: ## Override the top-level when. - # comments: - # - "/test filebeat for windows-7" - # labels: - # - "windows-7" - # branches: true ## for all the branches - # tags: true ## for all the tags + # stage: extended + #windows-7-32: See https://github.com/elastic/beats/issues/19831 + # mage: "mage build unitTest" + # platforms: ## override default labels in this specific stage. + # - "windows-7-32-bit" + # stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml index 50c95bee667..107e15107c3 100644 --- a/deploy/kubernetes/Jenkinsfile.yml +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -13,8 +13,10 @@ when: platform: "immutable && ubuntu-18" ## default label for all the stages stages: lint: - make: | - make -C deploy/kubernetes all; - make check-no-changes; + make: | + make -C deploy/kubernetes all; + make check-no-changes; + stage: lint k8sTest: k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" + stage: mandatory diff --git a/dev-tools/Jenkinsfile.yml b/dev-tools/Jenkinsfile.yml index e1970328515..98c2f4db210 100644 --- a/dev-tools/Jenkinsfile.yml +++ b/dev-tools/Jenkinsfile.yml @@ -13,4 +13,5 @@ when: platform: "immutable && ubuntu-18" ## default label for all the stages stages: lint: - make: "make -C dev-tools check" + make: "make -C dev-tools check" + stage: lint diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index ac8899504ac..121abfbb6ba 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C filebeat check; make -C filebeat update; make -C x-pack/filebeat check; make -C x-pack/filebeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,9 +34,11 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,45 +52,48 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" #- "windows-2008-r2" https://github.com/elastic/beats/issues/19795 + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended #windows-7: See https://github.com/elastic/beats/issues/22317 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7" # when: ## Override the top-level when. - # comments: - # - "/test filebeat for windows-7" - # labels: - # - "windows-7" - # branches: true ## for all the branches - # tags: true ## for all the tags + # stage: packaging windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml index f3210ee482a..9cc3f948a73 100644 --- a/generator/Jenkinsfile.yml +++ b/generator/Jenkinsfile.yml @@ -17,8 +17,10 @@ platform: "immutable && ubuntu-18" ## default label for all the stages stages: metricbeat-test: make: "make -C generator/_templates/metricbeat test test-package" + stage: mandatory beat-test: make: "make -C generator/_templates/beat test test-package" + stage: mandatory macos-metricbeat: make: "make -C generator/_templates/metricbeat test" platforms: ## override default label in this specific stage. @@ -32,6 +34,7 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended macos-beat: make: "make -C generator/_templates/beat test" platforms: ## override default label in this specific stage. @@ -45,3 +48,4 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 5f4324820fe..b605fea3e8f 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C heartbeat check; make -C heartbeat update; make -C x-pack/heartbeat check; make -C x-pack/heartbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,8 +34,10 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -48,45 +51,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index a91d99dd115..17444786286 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -13,11 +13,12 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C journalbeat check; make -C journalbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -31,15 +32,19 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended unitTest: mage: "mage build unitTest" + stage: mandatory packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 42ca46b2afb..1e9f7724028 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -12,13 +12,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C libbeat check; make -C libbeat update; make -C x-pack/libbeat check; make -C x-pack/libbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -32,7 +33,10 @@ stages: - "armTest" build: mage: "mage build test" + stage: mandatory crosscompile: make: "make -C libbeat crosscompile" + stage: mandatory stress-tests: make: "make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests" + stage: mandatory diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 7d7b4a6227e..57f18ff81d2 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -13,23 +13,28 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C metricbeat check; make -C metricbeat update; make -C x-pack/metricbeat check; make -C x-pack/metricbeat update; make check-no-changes; + stage: lint unitTest: mage: "mage build unitTest" + stage: mandatory goIntegTest: mage: "mage goIntegTest" withModule: true + stage: mandatory pythonIntegTest: mage: "mage pythonIntegTest" withModule: true + stage: mandatory crosscompile: make: "make -C metricbeat crosscompile" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -47,41 +52,51 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false ## e2e is enabled only for x-pack beats + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 659b3878364..86cf53094b9 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -13,7 +13,7 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C packetbeat check; make -C packetbeat update; @@ -21,6 +21,7 @@ stages: cd x-pack/packetbeat; mage check; mage update; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -34,8 +35,10 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,45 +52,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index a66ffcf1c73..d90f7500946 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -13,45 +13,59 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C winlogbeat check; make -C winlogbeat update; make -C x-pack/winlogbeat check; make -C x-pack/winlogbeat update; make check-no-changes; + stage: lint crosscompile: make: "make -C winlogbeat crosscompile" + stage: mandatory windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - - "windows-2008-r2" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended + windows-2008: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 663cd82b161..73ebf4b28ec 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/auditbeat check; make -C x-pack/auditbeat update; make -C auditbeat check; make -C auditbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,9 +34,11 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage update build test" withModule: true ## run the ITs only if the changeset affects a specific module. + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,45 +52,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml index d31c4e32b5f..c608d5ed54f 100644 --- a/x-pack/dockerlogbeat/Jenkinsfile.yml +++ b/x-pack/dockerlogbeat/Jenkinsfile.yml @@ -13,21 +13,25 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/dockerlogbeat check; make -C x-pack/dockerlogbeat update; make check-no-changes; + stage: lint build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. + stage: mandatory packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index d85a96c71f6..7b45fe7ae7a 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -13,11 +13,12 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/elastic-agent check; make -C x-pack/elastic-agent update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -31,8 +32,10 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -46,52 +49,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended #windows-7-32: See https://github.com/elastic/beats/issues/22316 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7-32-bit" - # when: ## Override the top-level when. - # comments: - # - "/test x-pack/elastic-agent for windows-7-32" - # labels: - # - "windows-7-32" - # branches: true ## for all the branches - # tags: true ## for all the tags + # stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index cc13fe11bc3..1246581eb0d 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -13,13 +13,14 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/filebeat check; make -C x-pack/filebeat update; make -C filebeat check; make -C filebeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,9 +34,11 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,52 +52,56 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended #windows-7-32: See https://github.com/elastic/beats/issues/22315 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7-32-bit" - # when: ## Override the top-level when. - # comments: - # - "/test x-pack/filebeat for windows-7-32" - # labels: - # - "windows-7-32" - # branches: true ## for all the branches - # tags: true ## for all the tags + # stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - - "arm" \ No newline at end of file + - "arm" + stage: packaging \ No newline at end of file diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 1dede6243e5..8128d9d716f 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -13,11 +13,12 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/functionbeat check; make -C x-pack/functionbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -31,6 +32,7 @@ stages: - "armTest" build: mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -44,39 +46,49 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging diff --git a/x-pack/heartbeat/Jenkinsfile.yml b/x-pack/heartbeat/Jenkinsfile.yml index ef2429d3dec..bdf8b8f9817 100644 --- a/x-pack/heartbeat/Jenkinsfile.yml +++ b/x-pack/heartbeat/Jenkinsfile.yml @@ -13,15 +13,17 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/heartbeat check; make -C x-pack/heartbeat update; make -C heartbeat check; make -C heartbeat update; make check-no-changes; + stage: lint build: mage: "mage build test" + stage: mandatory macos: mage: "mage build test" platforms: ## override default label in this specific stage. @@ -35,6 +37,7 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended # TODO: there are windows test failures already reported # https://github.com/elastic/beats/issues/23957 and https://github.com/elastic/beats/issues/23958 # waiting for being fixed. @@ -42,41 +45,51 @@ stages: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-2019" +# stage: mandatory # windows-2016: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-2016" +# stage: extended # windows-2012: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-2012-r2" +# stage: extended # windows-10: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-10" +# stage: extended # windows-2008: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-2008-r2" +# stage: extended # windows-8: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-8" +# stage: extended # windows-7: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-7" +# stage: extended # windows-7-32: # mage: "mage build test" # platforms: ## override default labels in this specific stage. # - "windows-7-32-bit" +# stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 267cf1be9ac..5d295d73904 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -13,11 +13,12 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/libbeat check; make -C x-pack/libbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -31,5 +32,7 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 497fbc4b5eb..e82fd9baef6 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -13,15 +13,17 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/metricbeat check; make -C x-pack/metricbeat update; make -C metricbeat check; make -C metricbeat update; make check-no-changes; + stage: lint unitTest: mage: "mage build unitTest" + stage: mandatory cloud: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. @@ -34,6 +36,7 @@ stages: - "/test x-pack/metricbeat for aws cloud" labels: - "aws" + stage: extended macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -47,47 +50,58 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended # windows-7-32 builder has been disabled due to https://github.com/elastic/beats/issues/24337 #windows-7-32: # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7-32-bit" + # stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false entrypoint: 'metricbeat-test.sh' + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/osquerybeat/Jenkinsfile.yml b/x-pack/osquerybeat/Jenkinsfile.yml index d3c591e0d7e..7ed2ac5e088 100644 --- a/x-pack/osquerybeat/Jenkinsfile.yml +++ b/x-pack/osquerybeat/Jenkinsfile.yml @@ -13,13 +13,15 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/osquerybeat check; make -C x-pack/osquerybeat update; make check-no-changes; + stage: lint build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -33,35 +35,44 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index e35dfa34879..40227865b5f 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -13,7 +13,7 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - Lint: + lint: mage: | mage check; mage update; @@ -21,6 +21,7 @@ stages: make -C packetbeat check; make -C packetbeat update; make check-no-changes; + stage: lint arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -34,8 +35,10 @@ stages: - "armTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended build: mage: "mage build test" + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -49,46 +52,57 @@ stages: - "macosTest" branches: true ## for all the branches tags: true ## for all the tags + stage: extended windows: mage: "mage build unitTest" withModule: true platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false + stage: packaging packaging-arm: packaging-arm: "mage package" e2e: enabled: false platforms: ## override default label in this specific stage. - "arm" + stage: packaging diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index 8b03fed80a3..fe37a3f40a9 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -13,7 +13,7 @@ when: tags: true ## for all the tags platform: "windows-2019" ## default label for all the stages stages: - Lint: + lint: make: | make -C x-pack/winlogbeat check; make -C x-pack/winlogbeat update; @@ -22,42 +22,52 @@ stages: make check-no-changes; platforms: ## override default labels in this specific stage. - "immutable && ubuntu-18" + stage: lint build: mage: "mage build unitTest" withModule: true platforms: ## override default labels in this specific stage. - "windows-2019" + stage: mandatory windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" + stage: extended windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" + stage: extended windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" + stage: extended windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" + stage: extended windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" + stage: extended windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" + stage: extended windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" + stage: extended packaging-linux: packaging-linux: "mage package" e2e: enabled: false platforms: ## override default labels in this specific stage. - "immutable && ubuntu-18" + stage: packaging From 1fa1edf6a26a27e8bc68e78b07a05dc97fe04b7f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 15 Apr 2021 15:37:06 +0100 Subject: [PATCH 3/9] Use filterStage argument --- Jenkinsfile | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 353bea2a476..c0b083da89a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -105,7 +105,7 @@ pipeline { } } steps { - runBuildAndTest(filter: 'mandatory') + runBuildAndTest(filterStage: 'mandatory') } } stage('Extended') { @@ -124,7 +124,7 @@ pipeline { } } steps { - runBuildAndTest(filter: 'extended') + runBuildAndTest(filterStage: 'extended') } } stage('Packaging') { @@ -143,7 +143,7 @@ pipeline { } } steps { - runBuildAndTest(filter: 'packaging') + runBuildAndTest(filterStage: 'packaging') } } stage('Packaging-Pipeline') { @@ -187,10 +187,8 @@ def runLinting() { def mapParallelTasks = [:] def content = readYaml(file: 'Jenkinsfile.yml') content['projects'].each { projectName -> - generateStages(project: projectName, changeset: content['changeset']).each { k,v -> - if (v?.stage?.equals('lint')) { - mapParallelTasks["${k}"] = v - } + generateStages(project: projectName, changeset: content['changeset'], filterStage: 'lint').each { k,v -> + mapParallelTasks["${k}"] = v } } mapParallelTasks['default'] = { @@ -204,7 +202,7 @@ def runLinting() { } def runBuildAndTest(Map args = [:]) { - def filter = args.get('filter', 'mandatory') + def filterStage = args.get('filterStage', 'mandatory') deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${BASE_DIR}"){ @@ -214,10 +212,8 @@ def runBuildAndTest(Map args = [:]) { error 'Pull Request has been configured to be disabled when there is a skip-ci label match' } else { content['projects'].each { projectName -> - generateStages(project: projectName, changeset: content['changeset'], filter: args.filter).each { k,v -> - if (v?.stage?.equals(filter)) { - mapParallelTasks["${k}"] = v - } + generateStages(project: projectName, changeset: content['changeset'], filterStage: filterStage).each { k,v -> + mapParallelTasks["${k}"] = v } } notifyBuildReason() @@ -258,6 +254,7 @@ def getBranchIndice(String compare) { */ def generateStages(Map args = [:]) { def projectName = args.project + def filterStage = args.get('filterStage', 'all') def changeset = args.changeset def mapParallelStages = [:] def fileName = "${projectName}/Jenkinsfile.yml" @@ -265,7 +262,7 @@ def generateStages(Map args = [:]) { def content = readYaml(file: fileName) // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation. if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) { - mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this)) + mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this), filterStage: filterStage) } } else { log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") From 1b860dc5fcf6b0a08a193fb5392fdc1f816a4e63 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 15 Apr 2021 15:50:14 +0100 Subject: [PATCH 4/9] for testing purposes --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c0b083da89a..02ce820bad3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!/usr/bin/env groovy -@Library('apm@current') _ +@Library('apm@feature/filterStage') _ pipeline { agent { label 'ubuntu-18 && immutable' } From 938a7c0cd4042add81df95cb4548cb3527460d3f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Sun, 18 Apr 2021 13:23:58 +0100 Subject: [PATCH 5/9] Set env variable in the mandatory checkout stage --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 02ce820bad3..d5c5e4f1f33 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { setEnvVar('GO_VERSION', readFile(".go-version").trim()) withEnv(["HOME=${env.WORKSPACE}"]) { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } + setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) } } } @@ -77,7 +78,6 @@ pipeline { withGithubNotify(context: "Lint") { withBeatsEnv(archive: false, id: "lint") { dumpVariables() - setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) whenTrue(env.ONLY_DOCS == 'true') { cmd(label: "make check", script: "make check") } From 607b30e7909c88fa5dc2be221f3849b6a453c1a8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Sun, 18 Apr 2021 14:36:02 +0100 Subject: [PATCH 6/9] Split withBeatsEnv in env and prepare context --- Jenkinsfile | 119 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 51 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d5c5e4f1f33..c604d8452e6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -61,7 +61,7 @@ pipeline { setEnvVar('GO_MOD_CHANGES', isGitRegionMatch(patterns: [ '^go.mod' ], shouldMatchAll: false).toString()) setEnvVar('PACKAGING_CHANGES', isGitRegionMatch(patterns: [ '^dev-tools/packaging/.*' ], shouldMatchAll: false).toString()) setEnvVar('GO_VERSION', readFile(".go-version").trim()) - withEnv(["HOME=${env.WORKSPACE}"]) { + withBeatsEnv() { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) } @@ -76,7 +76,7 @@ pipeline { steps { stageStatusCache(id: 'Lint'){ withGithubNotify(context: "Lint") { - withBeatsEnv(archive: false, id: "lint") { + withBeatsContext(archive: false, id: "lint") { dumpVariables() whenTrue(env.ONLY_DOCS == 'true') { cmd(label: "make check", script: "make check") @@ -290,7 +290,7 @@ def k8sTest(Map args = [:]) { stage("${args.context} ${v}"){ withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ withGithubNotify(context: "${args.context} ${v}") { - withBeatsEnv(archive: false, withModule: false) { + withBeatsContext(archive: false, withModule: false) { retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") } retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") } try { @@ -540,7 +540,7 @@ def target(Map args = [:]) { def dockerArch = args.get('dockerArch', 'amd64') withNode(labels: args.label, sleepMin: 30, sleepMax: 200, forceWorkspace: true){ withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { + withBeatsContext(archive: true, withModule: withModule, directory: directory, id: args.id) { dumpVariables() // make commands use -C while mage commands require the dir(folder) // let's support this scenario with the location variable. @@ -565,15 +565,13 @@ def target(Map args = [:]) { } } + + /** -* This method wraps all the environment setup and pre-requirements to run any commands. +* This method wraps all the environment to run any commands. */ def withBeatsEnv(Map args = [:], Closure body) { - def archive = args.get('archive', true) - def withModule = args.get('withModule', false) - def directory = args.get('directory', '') - - def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags, userProfile + def goRoot, path, magefile, pythonEnv, gox_flags, userProfile if(isUnix()) { if (isArm() && is64arm()) { @@ -587,8 +585,6 @@ def withBeatsEnv(Map args = [:], Closure body) { path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" magefile = "${WORKSPACE}/.magefile" pythonEnv = "${WORKSPACE}/python-env" - testResults = '**/build/TEST*.xml' - artifacts = '**/build/TEST*.out' } else { // NOTE: to support Windows 7 32 bits the arch in the mingw and go context paths is required. def mingwArch = is32() ? '32' : '64' @@ -599,20 +595,9 @@ def withBeatsEnv(Map args = [:], Closure body) { goRoot = "${userProfile}\\.gvm\\versions\\go${GO_VERSION}.windows.${goArch}" path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};C:\\tools\\mingw${mingwArch}\\bin;${env.PATH}" magefile = "${env.WORKSPACE}\\.magefile" - testResults = "**\\build\\TEST*.xml" - artifacts = "**\\build\\TEST*.out" gox_flags = '-arch 386' } - // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace. - // Windows workers are ephemerals, so this should not really affect us. - if(isUnix()) { - deleteDir() - } - - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' withEnv([ "DOCKER_PULL=0", "GOPATH=${env.WORKSPACE}", @@ -620,7 +605,6 @@ def withBeatsEnv(Map args = [:], Closure body) { "GOX_FLAGS=${gox_flags}", "HOME=${env.WORKSPACE}", "MAGEFILE_CACHE=${magefile}", - "MODULE=${module}", "PATH=${path}", "PYTHON_ENV=${pythonEnv}", "RACE_DETECTOR=true", @@ -629,32 +613,65 @@ def withBeatsEnv(Map args = [:], Closure body) { "OLD_USERPROFILE=${env.USERPROFILE}", "USERPROFILE=${userProfile}" ]) { - if(isDockerInstalled()) { - dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - dockerLogin(secret: "${DOCKERHUB_SECRET}", registry: 'docker.io') - } - dir("${env.BASE_DIR}") { - installTools(args) - // Skip to upload the generated files by default. - def upload = false - try { - // Add more stability when dependencies are not accessible temporarily - // See https://github.com/elastic/beats/issues/21609 - // retry/try/catch approach reports errors, let's avoid it to keep the - // notifications cleaner. - if (cmd(label: 'Download modules to local cache', script: 'go mod download', returnStatus: true) > 0) { - cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true) - } - body() - } catch(err) { - // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage - upload = true - error("Error '${err.toString()}'") - } finally { - if (archive) { - archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) + body() + } +} + +/** +* This method wraps all the environment setup and pre-requirements to run any commands. +*/ +def withBeatsContext(Map args = [:], Closure body) { + def archive = args.get('archive', true) + def withModule = args.get('withModule', false) + def directory = args.get('directory', '') + + def testResults, artifacts + + if(isUnix()) { + testResults = '**/build/TEST*.xml' + artifacts = '**/build/TEST*.out' + // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace. + // Windows workers are ephemerals, so this should not really affect us. + deleteDir() + } else { + testResults = "**\\build\\TEST*.xml" + artifacts = "**\\build\\TEST*.out" + } + + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + // NOTE: This is required to run after the unstash + def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' + withEnv([ + "MODULE=${module}" + ]) { + withBeatsEnv(args) { + if(isDockerInstalled()) { + dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + dockerLogin(secret: "${DOCKERHUB_SECRET}", registry: 'docker.io') + } + dir("${env.BASE_DIR}") { + installTools(args) + // Skip to upload the generated files by default. + def upload = false + try { + // Add more stability when dependencies are not accessible temporarily + // See https://github.com/elastic/beats/issues/21609 + // retry/try/catch approach reports errors, let's avoid it to keep the + // notifications cleaner. + if (cmd(label: 'Download modules to local cache', script: 'go mod download', returnStatus: true) > 0) { + cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true) + } + body() + } catch(err) { + // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage + upload = true + error("Error '${err.toString()}'") + } finally { + if (archive) { + archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) + } + tearDown() } - tearDown() } } } @@ -838,7 +855,7 @@ def startCloudTestEnv(Map args = [:]) { def dirs = args.get('dirs',[]) stage("${name}-prepare-cloud-env"){ withCloudTestEnv() { - withBeatsEnv(archive: false, withModule: false) { + withBeatsContext(archive: false, withModule: false) { try { dirs?.each { folder -> retryWithSleep(retries: 2, seconds: 5, backoff: true){ @@ -880,7 +897,7 @@ def terraformCleanup(Map args = [:]) { String directory = args.dir stage("${name}-tear-down-cloud-env"){ withCloudTestEnv() { - withBeatsEnv(archive: false, withModule: false) { + withBeatsContext(archive: false, withModule: false) { unstash("terraform-${name}") retryWithSleep(retries: 2, seconds: 5, backoff: true) { sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") From 2e31c9ad496ecb08f5c39247edfbef3c71918ac1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Sun, 18 Apr 2021 15:17:30 +0100 Subject: [PATCH 7/9] Revert "Split withBeatsEnv in env and prepare context" This reverts commit 607b30e7909c88fa5dc2be221f3849b6a453c1a8. --- Jenkinsfile | 119 ++++++++++++++++++++++------------------------------ 1 file changed, 51 insertions(+), 68 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c604d8452e6..d5c5e4f1f33 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -61,7 +61,7 @@ pipeline { setEnvVar('GO_MOD_CHANGES', isGitRegionMatch(patterns: [ '^go.mod' ], shouldMatchAll: false).toString()) setEnvVar('PACKAGING_CHANGES', isGitRegionMatch(patterns: [ '^dev-tools/packaging/.*' ], shouldMatchAll: false).toString()) setEnvVar('GO_VERSION', readFile(".go-version").trim()) - withBeatsEnv() { + withEnv(["HOME=${env.WORKSPACE}"]) { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) } @@ -76,7 +76,7 @@ pipeline { steps { stageStatusCache(id: 'Lint'){ withGithubNotify(context: "Lint") { - withBeatsContext(archive: false, id: "lint") { + withBeatsEnv(archive: false, id: "lint") { dumpVariables() whenTrue(env.ONLY_DOCS == 'true') { cmd(label: "make check", script: "make check") @@ -290,7 +290,7 @@ def k8sTest(Map args = [:]) { stage("${args.context} ${v}"){ withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ withGithubNotify(context: "${args.context} ${v}") { - withBeatsContext(archive: false, withModule: false) { + withBeatsEnv(archive: false, withModule: false) { retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") } retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") } try { @@ -540,7 +540,7 @@ def target(Map args = [:]) { def dockerArch = args.get('dockerArch', 'amd64') withNode(labels: args.label, sleepMin: 30, sleepMax: 200, forceWorkspace: true){ withGithubNotify(context: "${context}") { - withBeatsContext(archive: true, withModule: withModule, directory: directory, id: args.id) { + withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { dumpVariables() // make commands use -C while mage commands require the dir(folder) // let's support this scenario with the location variable. @@ -565,13 +565,15 @@ def target(Map args = [:]) { } } - - /** -* This method wraps all the environment to run any commands. +* This method wraps all the environment setup and pre-requirements to run any commands. */ def withBeatsEnv(Map args = [:], Closure body) { - def goRoot, path, magefile, pythonEnv, gox_flags, userProfile + def archive = args.get('archive', true) + def withModule = args.get('withModule', false) + def directory = args.get('directory', '') + + def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags, userProfile if(isUnix()) { if (isArm() && is64arm()) { @@ -585,6 +587,8 @@ def withBeatsEnv(Map args = [:], Closure body) { path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" magefile = "${WORKSPACE}/.magefile" pythonEnv = "${WORKSPACE}/python-env" + testResults = '**/build/TEST*.xml' + artifacts = '**/build/TEST*.out' } else { // NOTE: to support Windows 7 32 bits the arch in the mingw and go context paths is required. def mingwArch = is32() ? '32' : '64' @@ -595,9 +599,20 @@ def withBeatsEnv(Map args = [:], Closure body) { goRoot = "${userProfile}\\.gvm\\versions\\go${GO_VERSION}.windows.${goArch}" path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};C:\\tools\\mingw${mingwArch}\\bin;${env.PATH}" magefile = "${env.WORKSPACE}\\.magefile" + testResults = "**\\build\\TEST*.xml" + artifacts = "**\\build\\TEST*.out" gox_flags = '-arch 386' } + // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace. + // Windows workers are ephemerals, so this should not really affect us. + if(isUnix()) { + deleteDir() + } + + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + // NOTE: This is required to run after the unstash + def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' withEnv([ "DOCKER_PULL=0", "GOPATH=${env.WORKSPACE}", @@ -605,6 +620,7 @@ def withBeatsEnv(Map args = [:], Closure body) { "GOX_FLAGS=${gox_flags}", "HOME=${env.WORKSPACE}", "MAGEFILE_CACHE=${magefile}", + "MODULE=${module}", "PATH=${path}", "PYTHON_ENV=${pythonEnv}", "RACE_DETECTOR=true", @@ -613,65 +629,32 @@ def withBeatsEnv(Map args = [:], Closure body) { "OLD_USERPROFILE=${env.USERPROFILE}", "USERPROFILE=${userProfile}" ]) { - body() - } -} - -/** -* This method wraps all the environment setup and pre-requirements to run any commands. -*/ -def withBeatsContext(Map args = [:], Closure body) { - def archive = args.get('archive', true) - def withModule = args.get('withModule', false) - def directory = args.get('directory', '') - - def testResults, artifacts - - if(isUnix()) { - testResults = '**/build/TEST*.xml' - artifacts = '**/build/TEST*.out' - // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace. - // Windows workers are ephemerals, so this should not really affect us. - deleteDir() - } else { - testResults = "**\\build\\TEST*.xml" - artifacts = "**\\build\\TEST*.out" - } - - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' - withEnv([ - "MODULE=${module}" - ]) { - withBeatsEnv(args) { - if(isDockerInstalled()) { - dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - dockerLogin(secret: "${DOCKERHUB_SECRET}", registry: 'docker.io') - } - dir("${env.BASE_DIR}") { - installTools(args) - // Skip to upload the generated files by default. - def upload = false - try { - // Add more stability when dependencies are not accessible temporarily - // See https://github.com/elastic/beats/issues/21609 - // retry/try/catch approach reports errors, let's avoid it to keep the - // notifications cleaner. - if (cmd(label: 'Download modules to local cache', script: 'go mod download', returnStatus: true) > 0) { - cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true) - } - body() - } catch(err) { - // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage - upload = true - error("Error '${err.toString()}'") - } finally { - if (archive) { - archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) - } - tearDown() + if(isDockerInstalled()) { + dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + dockerLogin(secret: "${DOCKERHUB_SECRET}", registry: 'docker.io') + } + dir("${env.BASE_DIR}") { + installTools(args) + // Skip to upload the generated files by default. + def upload = false + try { + // Add more stability when dependencies are not accessible temporarily + // See https://github.com/elastic/beats/issues/21609 + // retry/try/catch approach reports errors, let's avoid it to keep the + // notifications cleaner. + if (cmd(label: 'Download modules to local cache', script: 'go mod download', returnStatus: true) > 0) { + cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true) + } + body() + } catch(err) { + // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage + upload = true + error("Error '${err.toString()}'") + } finally { + if (archive) { + archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) } + tearDown() } } } @@ -855,7 +838,7 @@ def startCloudTestEnv(Map args = [:]) { def dirs = args.get('dirs',[]) stage("${name}-prepare-cloud-env"){ withCloudTestEnv() { - withBeatsContext(archive: false, withModule: false) { + withBeatsEnv(archive: false, withModule: false) { try { dirs?.each { folder -> retryWithSleep(retries: 2, seconds: 5, backoff: true){ @@ -897,7 +880,7 @@ def terraformCleanup(Map args = [:]) { String directory = args.dir stage("${name}-tear-down-cloud-env"){ withCloudTestEnv() { - withBeatsContext(archive: false, withModule: false) { + withBeatsEnv(archive: false, withModule: false) { unstash("terraform-${name}") retryWithSleep(retries: 2, seconds: 5, backoff: true) { sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") From 2491c2711aa714f17fa473a2d33cdb5135d549bd Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Sun, 18 Apr 2021 15:20:57 +0100 Subject: [PATCH 8/9] Use withMageEnv to get the Beats version --- Jenkinsfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index d5c5e4f1f33..fee30646fea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -63,6 +63,10 @@ pipeline { setEnvVar('GO_VERSION', readFile(".go-version").trim()) withEnv(["HOME=${env.WORKSPACE}"]) { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } + } + } + withMageEnv(version: "${env.GO_VERSION}"){ + dir("${BASE_DIR}"){ setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) } } From ea302e2a2d431a6d0a515bfc6eea3442ce132196 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 21 Apr 2021 11:35:07 +0100 Subject: [PATCH 9/9] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fee30646fea..087f6bd98e1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!/usr/bin/env groovy -@Library('apm@feature/filterStage') _ +@Library('apm@current') _ pipeline { agent { label 'ubuntu-18 && immutable' }