Merge remote-tracking branch 'upstream/master' into feature/reuse-pr-…

…automation

* upstream/master:
  Add searchIssuesWithFilters action for the lookForGitHubIssues (#1082)
  docs: update CHANGELOG.md
  [maven-release-plugin] prepare for next development iteration
  [maven-release-plugin] prepare release v1.1.211
  Fix build status and adjust Vault (#1074)
  Remove super-linter since it takes 20 minutes (#1083)

.ci/Jenkinsfile

@@ -201,17 +201,6 @@ pipeline {
         }
       }
     }
-    stage('Super-linter') {
-      steps {
-        withGithubNotify(context: 'Check super-linter', tab: 'tests') {
-          deleteDir()
-          unstash 'source'
-          dir("${BASE_DIR}"){
-            superLinter(failNever: true, junit: false)
-          }
-        }
-      }
-    }
     stage('Release') {
       environment {
         // gren and other tools are in the .ci/scripts folder.

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v1.1.211 (22/04/2021)
+
+#### ⚙️ CI
+
+-  Remove super-linter since it takes 20 minutes [#1083](https://github.com/elastic/apm-pipeline-library/pull/1083)
+-  Bump elastic stack version automation [#1081](https://github.com/elastic/apm-pipeline-library/pull/1081)
+
 ## v1.1.210 (20/04/2021)
 
 #### 🚀 Enhancements

pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>co.elastic</groupId>
   <artifactId>jenkins-library</artifactId>
-  <version>1.1.211-SNAPSHOT</version>
+  <version>1.1.212-SNAPSHOT</version>
   <name>APM Pipeline Shared Library</name>
   <description>Pipeline Shared Library containing utility steps.</description>
   <url>https://github.com/elastic/apm-pipeline-library</url>

src/test/groovy/GetVaultSecretStepTests.groovy

@@ -101,9 +101,19 @@ class GetVaultSecretStepTests extends ApmBasePipelineTest {
     assertTrue(assertMethodCallContainsPattern('error', 'getVaultSecret: Unable to get the secret.'))
   }
 
+  @Test
+  void testReadSecretWrapperWithParams() throws Exception {
+    script.readSecretWrapperWithParams(['role_id': 'dummy-role-id', 'secret_id': 'dummy-secret-id']) {
+     'dummy arg'
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('withCredentials', '[{credentialsId=vault-addr, variable=VAULT_ADDR}, {credentialsId=dummy-role-id, variable=VAULT_ROLE_ID}, {credentialsId=dummy-secret-id, variable=VAULT_SECRET_ID}]'))
+    assertJobStatusSuccess()
+  }
+
   @Test
   void testReadSecretWrapper() throws Exception {
-    script.readSecretWrapper {
+    script.readSecretWrapper() {
       // TODO
     }
     printCallStack()

src/test/groovy/LookForGitHubIssuesStepTests.groovy

@@ -40,25 +40,22 @@ class LookForGitHubIssuesStepTests extends ApmBasePipelineTest {
   void test_with_githubIssues_error() throws Exception {
     helper.registerAllowedMethod('githubIssues', [Map.class], { throw new Exception('unknown command "foo" for "gh issue"') })
     def ret = script.call(flakyList: [ 'test-foo' ])
-    printCallStack()
     assertTrue(ret['test-foo'].equals(''))
+    printCallStack()
   }
 
   @Test
-  void test_without_match() throws Exception {
-    helper.registerAllowedMethod('githubIssues', [Map.class], {
-      return [ 1 : [ state: 'OPEN', title: 'title' ]] })
-    def ret = script.call(flakyList: [ 'test-foo' ])
+  void test_flakySearch_without_match() throws Exception {
+    def issues = [ 1 : [ state: 'OPEN', title: 'title' ]]
+    def ret = script.searchFlakyIssues(flakyList: [ 'test-foo' ], issues: issues)
     printCallStack()
     assertTrue(ret['test-foo'].equals(''))
   }
 
   @Test
-  void test_with_match() throws Exception {
-    helper.registerAllowedMethod('githubIssues', [Map.class], {
-      return [ 1 : [ state: 'OPEN', title: 'title [test-foo]' ]] })
-    def ret = script.call(flakyList: [ 'test-foo' ])
-    println ret
+  void test_flakySearch_with_match() throws Exception {
+    def issues = [ 1 : [ state: 'OPEN', title: 'title [test-foo]' ]]
+    def ret = script.searchFlakyIssues(flakyList: [ 'test-foo' ], issues: issues)
     printCallStack()
     assertTrue(ret['test-foo'] == 1)
   }
@@ -67,8 +64,23 @@ class LookForGitHubIssuesStepTests extends ApmBasePipelineTest {
   void test_with_error() throws Exception {
     helper.registerAllowedMethod('githubIssues', [Map.class], { throw new Exception('force a failure') })
     def ret = script.call(flakyList: [ 'test-foo' ])
-    println ret
     printCallStack()
     assertTrue(ret.containsKey('test-foo'))
   }
+
+  @Test
+  void test_searchIssuesWithFilters_with_match() throws Exception {
+    def issues = [ 1 : [ state: 'OPEN', title: 'title [test-foo]' ]]
+    def ret = script.searchIssuesWithFilters(flakySearch: false, titleContains: 'title', issues: issues)
+    printCallStack()
+    assertTrue(ret.size() == 1)
+  }
+
+  @Test
+  void test_searchIssuesWithFilters_without_match() throws Exception {
+    def issues = [ 1 : [ state: 'OPEN', title: 'title [test-foo]' ]]
+    def ret = script.searchIssuesWithFilters(flakySearch: false, titleContains: 'acme', issues: issues)
+    printCallStack()
+    assertTrue(ret.size() == 0)
+  }
 }

vars/README.md

@@ -1734,17 +1734,24 @@ the log level by default is INFO.
 * `text`: Message to print. The color of the messages depends on the level.
 
 ## lookForGitHubIssues
-Look for all the open issues that were reported as flaky tests. It returns
+Look for all the open issues given some filters.
+
+For backward compatibilities the default behaviour uses the flaky tests. It returns
 a dictionary with the test-name as primary key and the github issue if any or empty otherwise.
 
 ```
   // Look for all the GitHub issues with label 'flaky-test' and test failures either test-foo or test-bar
-  lookForGitHubIssues( flakyList: [ 'test-foo', 'test-bar'], labelsFilter: [ 'flaky-test'])
+  lookForGitHubIssues(flakyList: [ 'test-foo', 'test-bar'], labelsFilter: [ 'flaky-test'])
+
+  // Look for all the GitHub issues with label 'automation' and the title contains 'bump: stack'
+  lookForGitHubIssues(flakySearch: false, labelsFilter: ['automation'], titleContains: 'bump: stack')
 ```
 
-* *flakyList*: list of test-failures. Mandatory
+* *flakySearch*: whether to run the default behaviour to look for flaky reported github issues. Optional. Default `true`
+* *flakyList*: list of test-failures. Optional. Default `[]`
 * *labelsFilter*: list of labels to be filtered when listing the GitHub issues. Optional
-* credentialsId: The credentials to access the repo (repo permissions). Optional. Default: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+* *titleContains*: title to be filtered when listing the GitHub issues. Optional
+* *credentialsId*: The credentials to access the repo (repo permissions). Optional. Default: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
 
 _NOTE_: Windows is not supported yet.
 

vars/getVaultSecret.groovy

@@ -24,7 +24,9 @@ import net.sf.json.JSONObject
 */
 def call(Map args = [:]){
   def secret = args.containsKey('secret') ? args.secret : error("getVaultSecret: No valid secret to looking for.")
-  return readSecret(secret)
+  def role_id = args.containsKey('role_id') ? args.role_id : 'vault-role-id'
+  def secret_id = args.containsKey('secret_id') ? args.secret_id : 'vault-secret-id'
+  return readSecret(secret, role_id, secret_id)
 }
 
 /**
@@ -39,13 +41,13 @@ def call(secret) {
     error("getVaultSecret: No valid secret to looking for.")
   }
   secret = 'secret/apm-team/ci/' + secret
-  return readSecret(secret)
+  return readSecret(secret, 'vault-role-id', 'vault-secret-id')
 }
 
-def readSecret(secret) {
+def readSecret(secret, role_id, secret_id) {
   def props = null
   log(level: 'INFO', text: 'getVaultSecret: Getting secrets')
-  readSecretWrapper() {
+  readSecretWrapperWithParams(['role_id': role_id, 'secret_id': secret_id]) {
     // When running in the CI with multiple parallel stages
     // the access could be considered as a DDOS attack. Let's sleep a bit if it fails.
     retryWithSleep(retries: 3, seconds: 5, backoff: true) {
@@ -58,6 +60,22 @@ def readSecret(secret) {
   return props
 }
 
+def readSecretWrapperWithParams(args, body) {
+  def role_id = args?.role_id
+  def secret_id = args?.secret_id
+  withCredentials([
+    string(credentialsId: 'vault-addr', variable: 'VAULT_ADDR'),
+    string(credentialsId: role_id, variable: 'VAULT_ROLE_ID'),
+    string(credentialsId: secret_id, variable: 'VAULT_SECRET_ID')]) {
+      withEnv([
+        "VAULT_AUTH_METHOD=approle", //Used by Ansible Vault modules
+        "VAULT_AUTHTYPE=approle" //Used by Ansible Vault modules
+      ]){
+        body()
+      }
+  }
+}
+
 def readSecretWrapper(body) {
   withCredentials([
     string(credentialsId: 'vault-addr', variable: 'VAULT_ADDR'),

vars/lookForGitHubIssues.groovy

@@ -16,46 +16,76 @@
 // under the License.
 
 /**
-  Look for all the open issues that were reported as flaky tests. It returns
+  Look for all the open issues given some filters.
+  
+  For backward compatibilities the default behaviour uses the flaky tests. It returns
   a dictionary with the test-name as primary key and the github issue if any or empty otherwise.
 
-  // Look for all the GitHub issues with label 'flaky-test' and test failures either test-foo or test-bar
-  lookForGitHubIssues( flakyList: [ 'test-foo', 'test-bar'], labelsFilter: [ 'flaky-test'])
+  // Look for all the open GitHub issues with label 'flaky-test' and test failures either test-foo or test-bar
+  lookForGitHubIssues(flakyList: [ 'test-foo', 'test-bar'], labelsFilter: [ 'flaky-test'])
+
+  // Look for all the open GitHub issues with label 'automation' and the title contains 'bump: stack'
+  lookForGitHubIssues(flakySearch: false, labelsFilter: ['automation'], titleContains: 'bump: stack')
 */
 def call(Map args = [:]) {
+  def flakySearch = args.get('flakySearch', true)
   def flakyList = args.get('flakyList', [])
   def labels = args.get('labelsFilter', [])
   def credentialsId = args.get('credentialsId', '2a9602aa-ab9f-4e52-baf3-b71ca88469c7')
+  def issues = [:]
+  try {
+    // Filter all the issues given those labels.
+    issues = githubIssues(labels: labels, credentialsId: credentialsId)
+  } catch(err) {
+    issues = [:]
+  } finally {
+    if (flakySearch) {
+      return searchFlakyIssues(flakyList: flakyList, issues: issues)
+    } else {
+      return searchIssuesWithFilters(issues: issues, titleContains: args.get('titleContains', ''))
+    }
+  }
+}
+
+def searchIssuesWithFilters(Map args = [:]) {
+  def titleContains = args.titleContains
+  def issues = args.issues
   def output = [:]
+  try {
+    output = issues?.findAll { issue, data -> data.title?.contains(titleContains) }
+  } catch (err) {
+    log(level: 'DEBUG', text: "search: failed with error '${err.toString()}'")
+  }
+  log(level: 'DEBUG', text: "search: output ${output}.")
+  return output
+}
+
+def searchFlakyIssues(Map args = [:]) {
+  def flakyList = args.flakyList
+  def issues = args.issues
   if (flakyList) {
-    try {
-      // Filter all the issues given those labels.
-      def issues = githubIssues(labels: labels, credentialsId: credentialsId)
-      if (issues) {
-        // for all the test failures and and github issues, let's look for the ones with
-        // the test-name in the issue title
-        flakyList.each { testName ->
-          def issue = issues.find { issue, data -> data.title?.contains(testName) }
-          if(issue) {
-            log(level: 'DEBUG', text: "lookForGitHubIssues: issue ${issue.key} matches ${testName}.")
-            output[testName] = issue.key
-          } else {
-            log(level: 'DEBUG', text: "lookForGitHubIssues: no match for ${testName}.")
-            output[testName] = ''
-          }
+    def output = [:]
+    if (issues) {
+      // for all the test failures and and github issues, let's look for the ones with
+      // the test-name in the issue title
+      flakyList.each { testName ->
+        def issue = issues.find { issue, data -> data.title?.contains(testName) }
+        if(issue) {
+          log(level: 'DEBUG', text: "flakySearch: issue ${issue.key} matches ${testName}.")
+          output[testName] = issue.key
+        } else {
+          log(level: 'DEBUG', text: "flakySearch: no match for ${testName}.")
+          output[testName] = ''
         }
-      } else {
-        // no issues could be found, let's report the list of test failures without any issue details.
-        flakyList.each { output.put(it, '') }
       }
-    } catch (err) {
+    } else {
       // no issues could be found, let's report the list of test failures without any issue details.
       flakyList.each { output.put(it, '') }
     }
-    log(level: 'DEBUG', text: "lookForGitHubIssues: output ${output}.")
+    log(level: 'DEBUG', text: "flakySearch: output ${output}.")
     return output
   } else {
-    log(level: 'WARN', text: "lookForGitHubIssues: flakyList is empty.")
-    return output
+    log(level: 'WARN', text: "flakySearch: flakyList is empty.")
+    return [:]
   }
 }

vars/lookForGitHubIssues.txt

@@ -1,13 +1,20 @@
-Look for all the open issues that were reported as flaky tests. It returns
+Look for all the open issues given some filters.
+
+For backward compatibilities the default behaviour uses the flaky tests. It returns
 a dictionary with the test-name as primary key and the github issue if any or empty otherwise.
 
 ```
   // Look for all the GitHub issues with label 'flaky-test' and test failures either test-foo or test-bar
-  lookForGitHubIssues( flakyList: [ 'test-foo', 'test-bar'], labelsFilter: [ 'flaky-test'])
+  lookForGitHubIssues(flakyList: [ 'test-foo', 'test-bar'], labelsFilter: [ 'flaky-test'])
+
+  // Look for all the GitHub issues with label 'automation' and the title contains 'bump: stack'
+  lookForGitHubIssues(flakySearch: false, labelsFilter: ['automation'], titleContains: 'bump: stack')
 ```
 
-* *flakyList*: list of test-failures. Mandatory
+* *flakySearch*: whether to run the default behaviour to look for flaky reported github issues. Optional. Default `true`
+* *flakyList*: list of test-failures. Optional. Default `[]`
 * *labelsFilter*: list of labels to be filtered when listing the GitHub issues. Optional
-* credentialsId: The credentials to access the repo (repo permissions). Optional. Default: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+* *titleContains*: title to be filtered when listing the GitHub issues. Optional
+* *credentialsId*: The credentials to access the repo (repo permissions). Optional. Default: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
 
 _NOTE_: Windows is not supported yet.

vars/withSecretVault.groovy

@@ -30,11 +30,14 @@ def call(Map args = [:], Closure body) {
   def pass_variable = args?.pass_var_name
   def pass_key = args.containsKey('pass_key') ? args.pass_key : 'password'
 
+  def role_id = args.containsKey('role_id') ? args.role_id : 'vault-role-id'
+  def secret_id = args.containsKey('secret_id') ? args.secret_id : 'vault-secret-id'
+
   if (!secret || !user_variable || !pass_variable) {
     error "withSecretVault: Missing variables"
   }
 
-  def props = getVaultSecret(secret: secret)
+  def props = getVaultSecret(secret: secret, role_id: role_id, secret_id: secret_id)
   if(props?.errors){
     error "withSecretVault: Unable to get credentials from the vault: " + props.errors.toString()
   }