ci: improve linting speed (#22103)

* ci: improve linting speed

* fix: use multiple agents for the matrix

* fix: add python-env

* fix: set make folder parameter

* feat: move k8s check and dev-tools to their own stages

* fix: lint in a single stage is faster

* fix: suggested changes

* chore:change triggers

* chore: update for packetbeat

Jenkinsfile

@@ -74,10 +74,12 @@ pipeline {
         GOFLAGS = '-mod=readonly'
       }
       steps {
-        withGithubNotify(context: 'Lint') {
-          withBeatsEnv(archive: false, id: 'lint') {
+        withGithubNotify(context: "Lint") {
+          withBeatsEnv(archive: false, id: "lint") {
             dumpVariables()
-            cmd(label: 'make check', script: 'make check')
+            cmd(label: "make check-python", script: "make check-python")
+            cmd(label: "make check-go", script: "make check-go")
+            cmd(label: "Check for changes", script: "make check-no-changes")
           }
         }
       }
@@ -496,7 +498,7 @@ def terraformApply(String directory) {
 }
 
 /**
-* Tear down the terraform environments, by looking for all terraform states in directory 
+* Tear down the terraform environments, by looking for all terraform states in directory
 * then it runs terraform destroy for each one.
 * It uses terraform states previously stashed by startCloudTestEnv.
 */

Jenkinsfile.yml

@@ -17,6 +17,7 @@ projects:
     - "x-pack/libbeat"
     - "x-pack/metricbeat"
     - "x-pack/winlogbeat"
+    - "dev-tools"
     ##- "x-pack/heartbeat"    It's not yet in the 1.0 pipeline.
     ##- "x-pack/journalbeat"  It's not yet in the 1.0 pipeline.
     ##- "x-pack/packetbeat"   It's not yet in the 1.0 pipeline.
@@ -45,4 +46,4 @@ disabled:
         labels:      ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR.
             - skip-ci
         ## TODO: This will allow to configure what to do based on the PR configuration
-        draft: true  ## Skip the GitHub Pull Request builds with Draft PRs.
+        draft: true  ## Skip the GitHub Pull Request builds with Draft PRs.

Makefile

@@ -94,20 +94,35 @@ clean: mage
 
 ## check : TBD.
 .PHONY: check
-check: python-env
+check:
 	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
-	@$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
-	@$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false)
+	$(MAKE) check-python
 	# check if vendor folder does not exists
 	[ ! -d vendor ]
-	@# Validate that all updates were committed
+	# Validate that all updates were committed
 	@$(MAKE) update
 	@$(MAKE) check-headers
-	go mod tidy
+	@$(MAKE) check-go
+	@$(MAKE) check-no-changes
+
+## ccheck-go : Check there is no changes in Go modules.
+.PHONY: check-go
+check-go:
+	@go mod tidy
+
+## ccheck-no-changes : Check there is no local changes.
+.PHONY: check-no-changes
+check-no-changes:
 	@git diff | cat
 	@git update-index --refresh
 	@git diff-index --exit-code HEAD --
 
+## check-python : Python Linting.
+.PHONY: check-python
+check-python: python-env
+	@$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
+	@$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false)
+
 ## check-headers : Check the license headers.
 .PHONY: check-headers
 check-headers: mage

auditbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C auditbeat check;
+          make -C auditbeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

deploy/kubernetes/Jenkinsfile.yml

@@ -2,6 +2,7 @@ when:
     branches: true             ## for all the branches
     changeset:                 ## when PR contains any of those entries in the changeset
         - "^deploy/kubernetes/.*"
+        - "^libbeat/docs/version.asciidoc"
     comments:                  ## when PR comment contains any of those entries
         - "/test deploy/kubernetes"
     labels:                    ## when PR labels matches any of those entries
@@ -11,5 +12,7 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    lint:
+       make: "make -C deploy/kubernetes all"
     k8sTest:
         k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10"

dev-tools/Jenkinsfile.yml

@@ -0,0 +1,16 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^dev-tools/.*"
+        - "^libbeat/scripts/Makefile"
+    comments:                  ## when PR comment contains any of those entries
+        - "/test dev-tools"
+    labels:                    ## when PR labels matches any of those entries
+        - "dev-tools"
+    parameters:                ## when parameter was selected in the UI.
+        - "dev-tools"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    lint:
+       make: "make -C dev-tools check"

filebeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C filebeat check;
+          make -C filebeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

heartbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C heartbeat check;
+          make -C heartbeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.
@@ -78,4 +83,3 @@ stages:
                 - "windows-10"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
-

journalbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C journalbeat check;
+          make -C journalbeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

libbeat/Jenkinsfile.yml

@@ -12,6 +12,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C libbeat check;
+          make -C libbeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

metricbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C metricbeat check;
+          make -C metricbeat update;
+          make check-no-changes;
     unitTest:
         mage: "mage build unitTest"
     goIntegTest:

packetbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C packetbeat check;
+          make -C packetbeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

winlogbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C winlogbeat check;
+          make -C winlogbeat update;
+          make check-no-changes;
     crosscompile:
         make: "make -C winlogbeat crosscompile"
     windows:

x-pack/auditbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/auditbeat check;
+          make -C x-pack/auditbeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

x-pack/dockerlogbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/dockerlogbeat check;
+          make -C x-pack/dockerlogbeat update;
+          make check-no-changes;
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

x-pack/elastic-agent/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/elastic-agent check;
+          make -C x-pack/elastic-agent update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

x-pack/filebeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/filebeat check;
+          make -C x-pack/filebeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

x-pack/functionbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/functionbeat check;
+          make -C x-pack/functionbeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

x-pack/libbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/libbeat check;
+          make -C x-pack/libbeat update;
+          make check-no-changes;
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

x-pack/metricbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/metricbeat check;
+          make -C x-pack/metricbeat update;
+          make check-no-changes;
     build:
         cloud: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

x-pack/packetbeat/Jenkinsfile.yml

@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        mage: |
+          mage check;
+          mage update;
+        make: "make check-no-changes"
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.

x-pack/winlogbeat/Jenkinsfile.yml

@@ -13,6 +13,13 @@ when:
     tags: true                 ## for all the tags
 platform: "windows-2019"       ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/winlogbeat check;
+          make -C x-pack/winlogbeat update;
+          make check-no-changes;
+        platforms:             ## override default labels in this specific stage.
+            - "linux && ubuntu-18"
     build:
         mage: "mage build unitTest"
         withModule: true