chore: move skip responsibility to the automation, not the CI (elasti…

…c#528)

* chore: move skip responsibility to the automation, not the CI

* chore: support skipping scenarios from Jenkins UI

.ci/.e2e-tests.yaml

@@ -7,11 +7,11 @@ SUITES:
   - suite: "helm"
     tags: "metricbeat"
   - suite: "fleet"
-    tags: "agent_endpoint_integration && ~skip"
+    tags: "agent_endpoint_integration"
   - suite: "fleet"
-    tags: "stand_alone_agent && ~skip"
+    tags: "stand_alone_agent"
   - suite: "fleet"
-    tags: "fleet_mode_agent && ~skip"
+    tags: "fleet_mode_agent"
   - suite: "metricbeat"
     tags: "integrations && apache"
   - suite: "metricbeat"

.ci/Jenkinsfile

@@ -27,6 +27,7 @@ pipeline {
     issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
   }
   parameters {
+    booleanParam(name: "SKIP_SCENARIOS", defaultValue: true, description: "If it's needed to skip those scenarios marked as @skip. Default true")
     string(name: 'runTestsSuites', defaultValue: '', description: 'A comma-separated list of test suites to run (default: empty to run all test suites)')
     booleanParam(name: "forceSkipGitChecks", defaultValue: false, description: "If it's needed to check for Git changes to filter by modified sources")
     booleanParam(name: "forceSkipPresubmit", defaultValue: false, description: "If it's needed to execute the pre-submit tests: unit and precommit.")
@@ -56,6 +57,7 @@ pipeline {
         HOME = "${env.WORKSPACE}"
         PATH = "${env.PATH}:${env.WORKSPACE}/bin:${env.WORKSPACE}/${env.BASE_DIR}/.ci/scripts"
         GO111MODULE = 'on'
+        SKIP_SCENARIOS = "${params.SKIP_SCENARIOS}"
         SLACK_CHANNEL = "${params.SLACK_CHANNEL.trim()}"
         ELASTIC_AGENT_DOWNLOAD_URL = "${params.ELASTIC_AGENT_DOWNLOAD_URL.trim()}"
         ELASTIC_AGENT_VERSION = "${params.ELASTIC_AGENT_VERSION.trim()}"

e2e/Makefile

@@ -5,12 +5,18 @@ FORMAT?=pretty
 LOG_INCLUDE_TIMESTAMP?=TRUE
 LOG_LEVEL?=INFO
 TIMEOUT_FACTOR?=3
+#true by default, allowing developers to set SKIP_SCENARIOS=false
+SKIP_SCENARIOS?=true
 STACK_VERSION?=
 PICKLES_VERSION?="2.20.1"
 VERSION_VALUE=`cat ../cli/VERSION.txt`
 
 ifneq ($(TAGS),)
 TAGS_FLAG=--tags
+ifeq ($(SKIP_SCENARIOS),true)
+## We always want to skip scenarios tagged with @skip
+TAGS+= && ~skip
+endif
 endif
 
 GO_IMAGE_TAG?='stretch'