Bump elastic stack version automation (#1081)

* Update .mergify.yml

* Bump elastic stack version automation

* Revert "Update .mergify.yml"

This reverts commit 7ead24b.

* Use the build_id

* Unrequired

* Support for the minor macro

* Use default location, and support the minor token

* Use matrix

* Revert "Use matrix"

This reverts commit 9eacd58.

* Revert "Revert "Use matrix""

This reverts commit d51fa28.

* Fix cosmetic changes in the UI

* Add support for the new parameters

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.ci/bumpStackVersion.groovy

@@ -0,0 +1,169 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import groovy.transform.Field
+
+@Library('apm@current') _
+
+// To store all the latest snapshot versions
+@Field def latestVersions
+
+pipeline {
+  agent { label 'linux && immutable' }
+  environment {
+    REPO = 'observability-dev'
+    HOME = "${env.WORKSPACE}"
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL='INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  parameters {
+    booleanParam(name: 'DRY_RUN_MODE', defaultValue: false, description: 'If true, allows to execute this pipeline in dry run mode, without sending a PR.')
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        git(credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: "https://github.com/elastic/${REPO}.git")
+      }
+    }
+    stage('Fetch latest versions') {
+      steps {
+        script {
+          latestVersions = artifactsApi(action: 'latest-versions')
+        }
+        archiveArtifacts 'latest-versions.json'
+      }
+    }
+    stage('Send Pull Request'){
+      options {
+        warnError('Pull Requests failed')
+      }
+      steps {
+        generateSteps()
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}
+
+def generateSteps(Map args = [:]) {
+  def projects = readYaml(file: '.ci/.bump-stack-version.yml')
+  def parallelTasks = [:]
+  projects['projects'].each { project ->
+    matrix( agent: 'linux && immutable',
+            axes:[
+              axis('REPO', [project.repo]),
+              axis('BRANCH', project.branches),
+              axis('ENABLED', [project.get('enabled', true)])
+            ],
+            excludes: [ axis('ENABLED', [ false ]) ]
+    ) {
+      bumpStackVersion(repo: env.REPO,
+                       scriptFile: "${project.script}",
+                       branch: env.BRANCH,
+                       reusePullRequest: project.get('reusePullRequest', false),
+                       labels: project.get('labels', ''))
+    }
+  }
+}
+
+def bumpStackVersion(Map args = [:]){
+  def repo = args.containsKey('repo') ? args.get('repo') : error('bumpStackVersion: repo argument is required')
+  def scriptFile = args.containsKey('scriptFile') ? args.get('scriptFile') : error('bumpStackVersion: scriptFile argument is required')
+  def branch = args.containsKey('branch') ? args.get('branch') : error('bumpStackVersion: branch argument is required')
+  def reusePullRequest = args.get('reusePullRequest', false)
+  def labels = args.get('labels', '')
+  log(level: 'INFO', text: "bumpStackVersion(repo: ${repo}, branch: ${branch}, scriptFile: ${scriptFile}, reusePullRequest: ${reusePullRequest}, labels: '${labels}')")
+
+  def branchName = findBranch(branch: branch, versions: latestVersions)
+  def versionEntry = latestVersions.get(branchName)
+  def message = createPRDescription(versionEntry)
+
+  catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
+    deleteDir()
+    setupAPMGitEmail(global: true)
+    git(url: "https://github.com/elastic/${repo}.git", branch: branchName, credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken')
+    sh(script: "${scriptFile} '${versionEntry.build_id}'", label: "Prepare changes for ${repo}")
+
+    pullRequest(reusePullRequest: reusePullRequest,
+                stackVersion: versionEntry.build_id,
+                message: message,
+                labels: labels)
+  }
+}
+
+def pullRequest(Map args = [:]){
+  def stackVersion = args.stackVersion
+  def message = args.message
+  def labels = args.labels.replaceAll('\\s','')
+  def reusePullRequest = args.get('reusePullRequest', false)
+  if (labels.trim()) {
+    labels = "automation,${labels}"
+  }
+
+  if (params.DRY_RUN_MODE) {
+    log(level: 'INFO', text: "DRY-RUN: pullRequest(stackVersion: ${stackVersion}, reusePullRequest: ${reusePullRequest}, labels: ${labels}, message: '${message}')")
+    return
+  }
+
+  if (reusePullRequest && ammendPullRequestIfPossible()) {
+    log(level: 'INFO', text: 'Reuse existing Pull Request')
+    return
+  }
+  githubCreatePullRequest(title: "bump: stack version '${stackVersion}'",
+                          labels: "${labels}", description: "${message}")
+}
+
+def ammendPullRequestIfPossible() {
+  log(level: 'INFO', text: 'TBD')
+  return false
+}
+
+def findBranch(Map args = [:]){
+  def branch = args.branch
+  // special macro to look for the latest minor version
+  if (branch.contains('<minor>')) {
+    def parts = branch.split('\\.')
+    def major = parts[0]
+    branch = args.versions.collect{ k,v -> k }.findAll { it ==~ /${major}\.\d+/}.sort().last()
+  }
+  return branch
+}
+
+def createPRDescription(versionEntry) {
+  return """
+  ### What
+  Bump stack version with the latest one.
+  ### Further details
+  ```
+  ${versionEntry}
+  ```
+  """
+}

.ci/jobs/bump-stack-version-pipeline.yml

@@ -0,0 +1,25 @@
+---
+- job:
+    name: apm-shared/bump-stack-version-pipeline
+    display-name: Bump Stack Version
+    description: Send PRs to the subscribed observability repositories if a new stack snapshot version has been released.
+    view: APM-CI
+    project-type: pipeline
+    parameters:
+      - string:
+          name: branch_specifier
+          default: master
+          description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/bumpStackVersion.groovy
+      scm:
+        - git:
+            url: git@github.com:elastic/apm-pipeline-library.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/apm-pipeline-library.git
+            branches:
+              - $branch_specifier

.ci/schedule-daily.groovy

@@ -81,6 +81,14 @@ pipeline {
           wait: false
         )
 
+        build(job: 'apm-shared/bump-stack-version-pipeline',
+          parameters: [
+            booleanParam(name: 'DRY_RUN_MODE', value: false)
+          ],
+          propagate: false,
+          wait: false
+        )
+
         build(job: 'apm-ui/apm-ui-e2e-tests-mbp/master',
           parameters: [
             booleanParam(name: 'FORCE', value: true),

resources/scripts/artifacts-api-latest-versions.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# It queries the artifacts-api entry point to fetch all the existing
+# snapshot versions and their metadata
+# It prints the output in the console in addition to the file
+# latest-versions.json
+#
+# Since it prints the output in the console avoid any cosmetic changes
+# with echo or set -x
+#
+set -eo pipefail
+
+URL="https://artifacts-api.elastic.co/v1"
+OUTPUT=latest-versions.json
+
+QUERY_OUTPUT=$(curl -s ${URL}/versions | jq -r '.aliases[] | select(contains("SNAPSHOT"))')
+LENGTH=$(echo $QUERY_OUTPUT | wc -l)
+i=0
+echo "{" > "${OUTPUT}"
+for version in ${QUERY_OUTPUT}; do
+    ## Array separator
+    i=$(($i + 1))
+    comma=""
+    if [ ${i} -gt 1 ] ; then
+        comma=","
+    elif  [ ${i} -ge ${LENGTH} ] ; then
+        comma=""
+    fi
+    LATEST_OUTPUT=$(curl -s ${URL}/versions/${version}/builds/latest | jq 'del(.build.projects,.manifests) | . |= .build')
+    BRANCH=$(echo $LATEST_OUTPUT | jq -r .branch)
+    {
+        echo "${comma}\"$BRANCH\":"
+        echo "${LATEST_OUTPUT}"
+    } >> "${OUTPUT}"
+done
+echo "}" >> "${OUTPUT}"
+
+cat "${OUTPUT}"

src/test/groovy/ArtifactsApiStepTests.groovy

@@ -0,0 +1,81 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Before
+import org.junit.Test
+import static org.junit.Assert.assertFalse
+import static org.junit.Assert.assertTrue
+
+class ArtifactsApiStepTests extends ApmBasePipelineTest {
+
+  @Override
+  @Before
+  void setUp() throws Exception {
+    super.setUp()
+    script = loadScript('vars/artifactsApi.groovy')
+  }
+
+  @Test
+  void testWindows() throws Exception {
+    testWindows() {
+      script.call()
+    }
+  }
+
+  @Test
+  void test_missing_action() throws Exception {
+    testMissingArgument('action') {
+      script.call()
+    }
+  }
+
+  @Test
+  void test_unsupported_action() throws Exception {
+    try {
+      script.call(action: 'unknown')
+    } catch(e) {
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', "artifactsApi: unsupported action."))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_latest_versions() throws Exception {
+    helper.registerAllowedMethod('sh', [Map.class],{'''
+    {
+      "6.8":
+        {
+          "release_branch": "6.8",
+          "branch": "6.8",
+          "version": "6.8.16-SNAPSHOT",
+          "build_id": "6.8.16-ac54b152"
+      },
+      "7.x":
+        {
+          "release_branch": "7.x",
+          "branch": "7.x",
+          "version": "7.13.0-SNAPSHOT",
+          "build_id": "7.13.0-a6f186fd"
+      }
+    }'''})
+    def obj = script.call(action: 'latest-versions')
+    assertTrue(obj.get('6.8').branch.equals('6.8'))
+    printCallStack()
+  }
+}

vars/README.md

@@ -84,6 +84,24 @@ by default it set the `APM_CLI_SERVICE_NAME` to the value of `JOB_NAME`
   pipelineManager([ apmTraces: [ when: 'ALWAYS' ] ])
 ```
 
+## artifactsApi
+This step helps to query the artifacts-api Rest API and returns
+ a JSON object.
+
+```
+import groovy.transform.Field
+
+@Field def latestVersions
+
+script {
+  versions = artifactsApi(action: 'latest-versions')
+}
+```
+
+* action: What's the action to be triggered. Mandatory
+
+_NOTE_: It only supports *nix.
+
 ## axis
 Build a vector of pairs [ name: "VAR_NAME", value: "VALUE" ]
 from a variable name (VAR_NAME) and a vector of values ([1,2,3,4,5]).