[CI] Pipeline 2.0 for monorepos

.ci/Jenkinsfile

@@ -0,0 +1,202 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'ubuntu-18 && immutable' }
+  environment {
+    AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
+    BASE_DIR = 'src/github.com/elastic/beats'
+    DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_COMPOSE_VERSION = "1.21.0"
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    GOX_FLAGS = "-arch amd64"
+    JOB_GCS_BUCKET = 'beats-ci-temp'
+    JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
+    PIPELINE_LOG_LEVEL = 'INFO'
+    OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
+    RUNBLD_DISABLE_NOTIFICATIONS = 'true'
+    TERRAFORM_VERSION = "0.12.24"
+    XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
+  }
+  options {
+    timeout(time: 2, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    quietPeriod(10)
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+  }
+  triggers {
+    issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)')
+  }
+  parameters {
+    booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
+    booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.')
+    string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
+    booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
+    booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
+    booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
+    booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
+  }
+  stages {
+    stage('Checkout') {
+      options { skipDefaultCheckout() }
+      steps {
+        pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true)
+        stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+        script {
+          dir("${BASE_DIR}"){
+            // Skip all the stages except docs for PR's with asciidoc and md changes only
+            env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true)
+          }
+        }
+      }
+    }
+    stage('Lint'){
+      options { skipDefaultCheckout() }
+      environment {
+        GOFLAGS = '-mod=readonly'
+      }
+      when {
+        anyOf {
+          not { changeRequest() }                           // If no PR
+          allOf {                                           // If PR and no docs changes
+            expression { return env.ONLY_DOCS == "false" }
+            changeRequest()
+          }
+          expression { return params.runAllStages }         // If UI forced
+        }
+      }
+      steps {
+        echo 'linting'
+      }
+    }
+    stage('Build&Test') {
+      options { skipDefaultCheckout() }
+      steps {
+        deleteDir()
+        unstash "source"
+        dir("${BASE_DIR}"){
+          script {
+            def masterFile = 'Jenkinsfile.yml'
+            def changeset = readYaml(file: masterFile)['changeset']
+            readYaml(file: masterFile)['projects'].each { project ->
+              generateSteps(name: project, changeset: changeset).each { k,v ->
+                mapParallelTasks["${k}"] = v
+              }
+            }
+            parallel(mapParallelTasks)
+          }
+        }
+      }
+    }
+  }
+  post {
+    // TODO: Support runbld
+    cleanup {
+      notifyBuildResult(prComment: true)
+    }
+  }
+}
+
+def generateSteps(Map args = [:]) {
+  def name = args.project
+  def changeset = args.changeset
+  def mapParallelStages = [:]
+  def masterFile = "${name}/Jenkinsfile.yml"
+  def content = readYaml(file: masterFile)
+  def defaultNode = content['platform']
+
+  if (whenValidation(project: name, content: content, changeset: changeset)) {
+    content['stages'].each { stageName ->
+      if (stageName['when']) {
+        if (whenValidation(project: "stageName", content: stageName['when'], changeset: changeset)) {
+          generateStages(defaultNode: defaultNode, content: stageName).each { k,v ->
+            mapParallelStages["${k}"] = v
+          }
+        }
+      } else {
+        generateStages(defaultNode: defaultNode, content: stageName).each { k,v ->
+          mapParallelStages["${k}"] = v
+        }
+      }
+  }
+  return mapParallelStages
+}
+
+def generateStages(Map args = [:]) {
+  def defaultNode = args.defaultNode
+  def mapParallelStages = [:]
+  if (content['platform']) {
+    content['platform']?.each {
+      mapParallelStages["${k}-${it}"] = generateStage(platform: it, content: stageName)
+    }
+  } else {
+    mapParallelStages["${k}"] = generateStage(platform: defaultNode, content: stageName)
+  }
+  return mapParallelStages
+}
+
+def generateStage(Map args = [:]) {
+  return {
+    node(it) {
+      sh "${content['command']}"
+    }
+  }
+}
+
+/**
+* Given the YAML definition and the changeset global macros
+* then it verifies if the project or stage should be enabled.
+*
+* It will generate a 
+*/
+def whenValidation(Map args = [:]) {
+  def project = args.project
+  def content = args.content
+  def patterns = args.changeset
+  def ret = false
+  if (content['comments']?.trim() && env.GITHUB_COMMENT?.trim()) {
+    if (content['comments']?.any { env.GITHUB_COMMENT?.toLowerCase()?.contains(it?.toLowerCase()) }) {
+      ret = true
+      markdownReason(project: project, reason: 'Comment is enabled and matches with the pattern.')
+    } else {
+      markdownReason(project: project, reason: 'Comment is enabled and does not match with the pattern.')
+    }
+  }
+  if (content['labels']) {
+    if (content['labels']?.any { matchesPrLabel(label: it) }) {
+      ret = true
+      markdownReason(project: project, reason: 'Label is enabled and matches with the pattern.')
+    } else {
+      markdownReason(project: project, reason: 'Label is enabled and does not match with the pattern.')
+    }
+  }
+  if (content['parameters']) {
+    if (content['parameters']?.any { params.[it] }) {
+      ret = true
+      markdownReason(project: project, reason: 'Parameter is enabled and matches with the pattern.')
+    } else {
+      markdownReason(project: project, reason: 'Parameter is enabled and does not match with the pattern.')
+    }
+  }
+  if (content['branches'] && env.BRANCH_NAME?.trim()) {
+    ret = true
+    markdownReason(project: project, reason: 'Branch is enabled and matches with the pattern.')
+  }
+  if (content['tags'] && env.TAG_NAME?.trim()) {
+    ret = true
+    markdownReason(project: project, reason: 'Tag is enabled and matches with the pattern.')
+  }
+  return ret
+}
+
+def markdownReason(Map args = [:]) {
+  echo "${args.project} - ${args.reason}"
+  // TODO create markdown
+}

.ci/jobs/beats-mbp-2.0.yml

@@ -0,0 +1,60 @@
+---
+- job:
+    name: Beats/beats-mbp-2.0
+    display-name: 'Beats (2.0)'
+    description: 'Beats Main Pipeline 2.0'
+    view: Beats
+    concurrent: true
+    project-type: multibranch
+    prune-dead-branches: true
+    days-to-keep: 30
+    script-path: '.ci/Jenkinsfile'
+    triggers: []
+    wrappers: []
+    scm:
+    - github:
+        branch-discovery: 'no-pr'
+        discover-pr-forks-strategy: 'merge-current'
+        discover-pr-forks-trust: 'permission'
+        discover-pr-origin: 'merge-current'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        discover-tags: true
+        disable-pr-notifications: true
+        notification-context: "beats-ci-2.0"
+        repo: 'beats'
+        repo-owner: 'elastic'
+        credentials-id: github-app-beats-ci
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - skip-initial-build: true
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: 365
+        - change-request:
+            ignore-target-only-changes: true
+        - named-branches:
+            - exact-name:
+                name: 'master'
+                case-sensitive: true
+            - regex-name:
+                regex: '7\.[x789]'
+                case-sensitive: true
+            - regex-name:
+                regex: '8\.\d+'
+                case-sensitive: true
+        clean:
+            after: true
+            before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: true

Jenkinsfile.yml

@@ -0,0 +1,29 @@
+projects:
+    - "auditbeat"
+    - "filebeat"
+    - "heartbeat"
+    - "journalbeat"
+    - "libbeat"
+    - "metricbeat"
+    - "packetbeat"
+    - "winlogbeat"
+    - "x-pack/auditbeat"
+    - "x-pack/dockerlogbeat"
+    - "x-pack/filebeat"
+    - "x-pack/functionbeat"
+    - "x-pack/heartbeat"
+    - "x-pack/journalbeat"
+    - "x-pack/libbeat"
+    - "x-pack/metricbeat"
+    - "x-pack/packetbeat"
+    - "x-pack/winlogbeat"
+
+## Changeset macros that are defined here and used in each specific 2.0 pipeline.
+changeset:
+    oss:
+        - "^Jenkinsfile"
+        - "^go.mod"
+        - "^libbeat/.*"
+        - "^testing/.*"
+        - "^dev-tools/.*"
+        - "^\\.ci/scripts/.*"

auditbeat/Jenkinsfile.yml

@@ -0,0 +1,44 @@
+when:
+    branches: true      ## for all the branches
+    tags: true          ## for all the tags
+    changeset:          ## when PR contains any of those entries in the changeset
+        - "auditbeat"
+        - '@oss'         ## special token regarding the changeset for the oss 
+    comments:
+        - "/test auditbeat"
+    labels:
+        - "auditbeat"
+    parameters:
+        - "auditbeat"
+platform:               ## default platform where the stages will use
+    - "linux && ubuntu-16"
+stages:
+    build:
+        command:
+            - "cd auditbeat && mage build test"
+    crosscompile: 
+        command:
+            - "make -C auditbeat crosscompile"
+    macos:
+        command:
+            - "cd auditbeat && mage build unitTest"
+        platform:
+          - "macos"
+        when:           ## Aggregate when with the top-level one.
+            comments:
+                - "/test auditbeat for macos"
+            labels:
+                - "macos"
+            parameters:
+                - "macos"
+    windows:
+        command:
+            - "cd auditbeat && mage build unitTest"
+        platform:
+            - "windows-2019"
+            - "windows-2016"
+        when:
+            comments:
+                - "/test auditbeat for windows"
+            parameters:
+                - "macos"