ci: improve linting speed (elastic#22103)

* ci: improve linting speed * fix: use multiple agents for the matrix * fix: add python-env * fix: set make folder parameter * feat: move k8s check and dev-tools to their own stages * fix: lint in a single stage is faster * fix: suggested changes * chore:change triggers * chore: update for packetbeat # Conflicts: # Jenkinsfile.yml # auditbeat/Jenkinsfile.yml # filebeat/Jenkinsfile.yml # heartbeat/Jenkinsfile.yml # journalbeat/Jenkinsfile.yml # libbeat/Jenkinsfile.yml # packetbeat/Jenkinsfile.yml # x-pack/auditbeat/Jenkinsfile.yml # x-pack/elastic-agent/Jenkinsfile.yml # x-pack/filebeat/Jenkinsfile.yml # x-pack/functionbeat/Jenkinsfile.yml # x-pack/libbeat/Jenkinsfile.yml # x-pack/packetbeat/Jenkinsfile.yml
kuisathaverat · Oct 28, 2020 · 37a1b34 · 37a1b34
1 parent a46a0ae
commit 37a1b34
Show file tree

Hide file tree

Showing 22 changed files with 162 additions and 10 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -74,10 +74,12 @@ pipeline {
         GOFLAGS = '-mod=readonly'
       }
       steps {
-        withGithubNotify(context: 'Lint') {
-          withBeatsEnv(archive: false, id: 'lint') {
+        withGithubNotify(context: "Lint") {
+          withBeatsEnv(archive: false, id: "lint") {
             dumpVariables()
-            cmd(label: 'make check', script: 'make check')
+            cmd(label: "make check-python", script: "make check-python")
+            cmd(label: "make check-go", script: "make check-go")
+            cmd(label: "Check for changes", script: "make check-no-changes")
           }
         }
       }
@@ -474,7 +476,7 @@ def terraformApply(String directory) {
 }
 
 /**
-* Tear down the terraform environments, by looking for all terraform states in directory 
+* Tear down the terraform environments, by looking for all terraform states in directory
 * then it runs terraform destroy for each one.
 * It uses terraform states previously stashed by startCloudTestEnv.
 */

diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml
@@ -17,6 +17,7 @@ projects:
     - "x-pack/libbeat"
     - "x-pack/metricbeat"
     - "x-pack/winlogbeat"
+    - "dev-tools"
     ##- "x-pack/heartbeat"    It's not yet in the 1.0 pipeline.
     ##- "x-pack/journalbeat"  It's not yet in the 1.0 pipeline.
     ##- "x-pack/packetbeat"   It's not yet in the 1.0 pipeline.

diff --git a/Makefile b/Makefile
@@ -94,20 +94,35 @@ clean: mage
 
 ## check : TBD.
 .PHONY: check
-check: python-env
+check:
 	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
-	@$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
-	@$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false)
+	$(MAKE) check-python
 	# check if vendor folder does not exists
 	[ ! -d vendor ]
-	@# Validate that all updates were committed
+	# Validate that all updates were committed
 	@$(MAKE) update
 	@$(MAKE) check-headers
-	go mod tidy
+	@$(MAKE) check-go
+	@$(MAKE) check-no-changes
+
+## ccheck-go : Check there is no changes in Go modules.
+.PHONY: check-go
+check-go:
+	@go mod tidy
+
+## ccheck-no-changes : Check there is no local changes.
+.PHONY: check-no-changes
+check-no-changes:
 	@git diff | cat
 	@git update-index --refresh
 	@git diff-index --exit-code HEAD --
 
+## check-python : Python Linting.
+.PHONY: check-python
+check-python: python-env
+	@$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
+	@$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false)
+
 ## check-headers : Check the license headers.
 .PHONY: check-headers
 check-headers: mage

diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
@@ -13,9 +13,14 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C auditbeat check;
+          make -C auditbeat update;
+          make check-no-changes;
     build:
         mage: "mage build test"
-    crosscompile: 
+    crosscompile:
         make: "make -C auditbeat crosscompile"
     macos:
         mage: "mage build unitTest"

diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml
@@ -2,6 +2,7 @@ when:
     branches: true             ## for all the branches
     changeset:                 ## when PR contains any of those entries in the changeset
         - "^deploy/kubernetes/.*"
+        - "^libbeat/docs/version.asciidoc"
     comments:                  ## when PR comment contains any of those entries
         - "/test deploy/kubernetes"
     labels:                    ## when PR labels matches any of those entries
@@ -11,5 +12,7 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    lint:
+       make: "make -C deploy/kubernetes all"
     k8sTest:
         k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10"
diff --git a/dev-tools/Jenkinsfile.yml b/dev-tools/Jenkinsfile.yml
@@ -0,0 +1,16 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^dev-tools/.*"
+        - "^libbeat/scripts/Makefile"
+    comments:                  ## when PR comment contains any of those entries
+        - "/test dev-tools"
+    labels:                    ## when PR labels matches any of those entries
+        - "dev-tools"
+    parameters:                ## when parameter was selected in the UI.
+        - "dev-tools"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    lint:
+       make: "make -C dev-tools check"
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C filebeat check;
+          make -C filebeat update;
+          make check-no-changes;
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C heartbeat check;
+          make -C heartbeat update;
+          make check-no-changes;
     build:
         mage: "mage build test"
     macos:

diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml
@@ -13,5 +13,10 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C journalbeat check;
+          make -C journalbeat update;
+          make check-no-changes;
     unitTest:
         mage: "mage build unitTest"
diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml
@@ -12,6 +12,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C libbeat check;
+          make -C libbeat update;
+          make check-no-changes;
     build:
         mage: "mage build test"
     crosscompile:

diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C metricbeat check;
+          make -C metricbeat update;
+          make check-no-changes;
     unitTest:
         mage: "mage build unitTest"
     goIntegTest:

diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C packetbeat check;
+          make -C packetbeat update;
+          make check-no-changes;
     build:
         mage: "mage build test"
     macos:

diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C winlogbeat check;
+          make -C winlogbeat update;
+          make check-no-changes;
     crosscompile:
         make: "make -C winlogbeat crosscompile"
     windows:

diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/auditbeat check;
+          make -C x-pack/auditbeat update;
+          make check-no-changes;
     build:
         mage: "mage update build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/dockerlogbeat check;
+          make -C x-pack/dockerlogbeat update;
+          make check-no-changes;
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.
diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/elastic-agent check;
+          make -C x-pack/elastic-agent update;
+          make check-no-changes;
     build:
         mage: "mage build test"
     macos:

diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/filebeat check;
+          make -C x-pack/filebeat update;
+          make check-no-changes;
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/functionbeat check;
+          make -C x-pack/functionbeat update;
+          make check-no-changes;
     build:
         mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions"
     macos:

diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml
@@ -13,5 +13,10 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/libbeat check;
+          make -C x-pack/libbeat update;
+          make check-no-changes;
     build:
         mage: "mage build test"
diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/metricbeat check;
+          make -C x-pack/metricbeat update;
+          make check-no-changes;
     build:
         cloud: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml
@@ -13,6 +13,39 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    Lint:
+        mage: |
+          mage check;
+          mage update;
+        make: "make check-no-changes"
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/packetbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
+    build:
+        mage: "mage build test"
+    macos:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "macosx"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/packetbeat for macos"
+            labels:
+                - "macOS"
+            parameters:
+                - "macosTest"
+            branches: true     ## for all the branches
+            tags: true
     windows:
         mage: "mage build unitTest"
         withModule: true

diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml
@@ -13,6 +13,13 @@ when:
     tags: true                 ## for all the tags
 platform: "windows-2019"       ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/winlogbeat check;
+          make -C x-pack/winlogbeat update;
+          make check-no-changes;
+        platforms:             ## override default labels in this specific stage.
+            - "linux && ubuntu-18"
     build:
         mage: "mage build unitTest"
         withModule: true