From 7d911b985ef0a9ac274a6cf49ffe95b46c7b130c Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 28 Oct 2020 13:09:12 +0100 Subject: [PATCH] ci: improve linting speed (#22103) * ci: improve linting speed * fix: use multiple agents for the matrix * fix: add python-env * fix: set make folder parameter * feat: move k8s check and dev-tools to their own stages * fix: lint in a single stage is faster * fix: suggested changes * chore:change triggers * chore: update for packetbeat # Conflicts: # x-pack/packetbeat/Jenkinsfile.yml --- Jenkinsfile | 8 ++++--- Jenkinsfile.yml | 3 ++- Makefile | 25 ++++++++++++++++----- auditbeat/Jenkinsfile.yml | 5 +++++ deploy/kubernetes/Jenkinsfile.yml | 3 +++ dev-tools/Jenkinsfile.yml | 16 ++++++++++++++ filebeat/Jenkinsfile.yml | 5 +++++ heartbeat/Jenkinsfile.yml | 6 ++++- journalbeat/Jenkinsfile.yml | 5 +++++ libbeat/Jenkinsfile.yml | 5 +++++ metricbeat/Jenkinsfile.yml | 5 +++++ packetbeat/Jenkinsfile.yml | 5 +++++ winlogbeat/Jenkinsfile.yml | 5 +++++ x-pack/auditbeat/Jenkinsfile.yml | 5 +++++ x-pack/dockerlogbeat/Jenkinsfile.yml | 5 +++++ x-pack/elastic-agent/Jenkinsfile.yml | 5 +++++ x-pack/filebeat/Jenkinsfile.yml | 5 +++++ x-pack/functionbeat/Jenkinsfile.yml | 5 +++++ x-pack/libbeat/Jenkinsfile.yml | 5 +++++ x-pack/metricbeat/Jenkinsfile.yml | 5 +++++ x-pack/packetbeat/Jenkinsfile.yml | 33 ++++++++++++++++++++++++++++ x-pack/winlogbeat/Jenkinsfile.yml | 7 ++++++ 22 files changed, 161 insertions(+), 10 deletions(-) create mode 100644 dev-tools/Jenkinsfile.yml diff --git a/Jenkinsfile b/Jenkinsfile index f90178d79da..4c8af6869f3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -75,10 +75,12 @@ pipeline { GOFLAGS = '-mod=readonly' } steps { - withGithubNotify(context: 'Lint') { - withBeatsEnv(archive: false, id: 'lint') { + withGithubNotify(context: "Lint") { + withBeatsEnv(archive: false, id: "lint") { dumpVariables() - cmd(label: 'make check', script: 'make check') + cmd(label: "make check-python", script: "make check-python") + cmd(label: "make check-go", script: "make check-go") + cmd(label: "Check for changes", script: "make check-no-changes") } } } diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index f7b21e1cbdf..3edd9a75d5c 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -17,6 +17,7 @@ projects: - "x-pack/libbeat" - "x-pack/metricbeat" - "x-pack/winlogbeat" + - "dev-tools" ##- "x-pack/heartbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/journalbeat" It's not yet in the 1.0 pipeline. ##- "x-pack/packetbeat" It's not yet in the 1.0 pipeline. @@ -45,4 +46,4 @@ disabled: labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. - skip-ci ## TODO: This will allow to configure what to do based on the PR configuration - draft: true ## Skip the GitHub Pull Request builds with Draft PRs. \ No newline at end of file + draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/Makefile b/Makefile index d64bb07776b..94e8391c295 100644 --- a/Makefile +++ b/Makefile @@ -94,20 +94,35 @@ clean: mage ## check : TBD. .PHONY: check -check: python-env +check: @$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;) - @$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) - @$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false) + $(MAKE) check-python # check if vendor folder does not exists [ ! -d vendor ] - @# Validate that all updates were committed + # Validate that all updates were committed @$(MAKE) update @$(MAKE) check-headers - go mod tidy + @$(MAKE) check-go + @$(MAKE) check-no-changes + +## ccheck-go : Check there is no changes in Go modules. +.PHONY: check-go +check-go: + @go mod tidy + +## ccheck-no-changes : Check there is no local changes. +.PHONY: check-no-changes +check-no-changes: @git diff | cat @git update-index --refresh @git diff-index --exit-code HEAD -- +## check-python : Python Linting. +.PHONY: check-python +check-python: python-env + @$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) + @$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false) + ## check-headers : Check the license headers. .PHONY: check-headers check-headers: mage diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index c68b5689f48..be9663b8e8d 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C auditbeat check; + make -C auditbeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml index 452771edfb5..e75e2f54c10 100644 --- a/deploy/kubernetes/Jenkinsfile.yml +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -2,6 +2,7 @@ when: branches: true ## for all the branches changeset: ## when PR contains any of those entries in the changeset - "^deploy/kubernetes/.*" + - "^libbeat/docs/version.asciidoc" comments: ## when PR comment contains any of those entries - "/test deploy/kubernetes" labels: ## when PR labels matches any of those entries @@ -11,5 +12,7 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + lint: + make: "make -C deploy/kubernetes all" k8sTest: k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" diff --git a/dev-tools/Jenkinsfile.yml b/dev-tools/Jenkinsfile.yml new file mode 100644 index 00000000000..28ead9ec813 --- /dev/null +++ b/dev-tools/Jenkinsfile.yml @@ -0,0 +1,16 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^dev-tools/.*" + - "^libbeat/scripts/Makefile" + comments: ## when PR comment contains any of those entries + - "/test dev-tools" + labels: ## when PR labels matches any of those entries + - "dev-tools" + parameters: ## when parameter was selected in the UI. + - "dev-tools" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + lint: + make: "make -C dev-tools check" diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index d2e1d866526..f13f0329345 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C filebeat check; + make -C filebeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 032ec411892..6135df2d26f 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C heartbeat check; + make -C heartbeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -78,4 +83,3 @@ stages: - "windows-10" branches: true ## for all the branches tags: true ## for all the tags - diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index 5715712dd4a..9358d6c5197 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C journalbeat check; + make -C journalbeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 692400e7253..0a59f7d4406 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -12,6 +12,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C libbeat check; + make -C libbeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index e6c4ffcef0e..993d0681469 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C metricbeat check; + make -C metricbeat update; + make check-no-changes; unitTest: mage: "mage build unitTest" goIntegTest: diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index ef373bb5f15..e956b7ddc6b 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C packetbeat check; + make -C packetbeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index 3b9c71bf0c3..a7620c47138 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C winlogbeat check; + make -C winlogbeat update; + make check-no-changes; crosscompile: make: "make -C winlogbeat crosscompile" windows: diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index f4e55ea6372..a65ccfeeecd 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C x-pack/auditbeat check; + make -C x-pack/auditbeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml index 703bb3d66a9..1eed20f80ed 100644 --- a/x-pack/dockerlogbeat/Jenkinsfile.yml +++ b/x-pack/dockerlogbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C x-pack/dockerlogbeat check; + make -C x-pack/dockerlogbeat update; + make check-no-changes; build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index d324e3381af..d1e415a6e78 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C x-pack/elastic-agent check; + make -C x-pack/elastic-agent update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index d28520b7c32..460115bbe41 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C x-pack/filebeat check; + make -C x-pack/filebeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 117d92e3179..5d8cf74c480 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C x-pack/functionbeat check; + make -C x-pack/functionbeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index ed22a8dfe70..9e5c382072d 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C x-pack/libbeat check; + make -C x-pack/libbeat update; + make check-no-changes; arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 8506eb12e69..2fe5da311fa 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -13,6 +13,11 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C x-pack/metricbeat check; + make -C x-pack/metricbeat update; + make check-no-changes; build: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index a3c11636dc6..422201a994a 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -13,6 +13,39 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + Lint: + mage: | + mage check; + mage update; + make: "make check-no-changes" + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/packetbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags + build: + mage: "mage build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Override the top-level when. + comments: + - "/test x-pack/packetbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + branches: true ## for all the branches + tags: true windows: mage: "mage build unitTest" withModule: true diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index 45dfcad9d45..f95e44395e1 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -13,6 +13,13 @@ when: tags: true ## for all the tags platform: "windows-2019" ## default label for all the stages stages: + Lint: + make: | + make -C x-pack/winlogbeat check; + make -C x-pack/winlogbeat update; + make check-no-changes; + platforms: ## override default labels in this specific stage. + - "linux && ubuntu-18" build: mage: "mage build unitTest" withModule: true