Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into feature/windows-7-64
Browse files Browse the repository at this point in the history 
* upstream/master: (93 commits)
  Update commands used in the quick start (elastic#22248)
  Add interval documentation to `monitor` metricset (elastic#22152)
  [CI] enable x-pack/packetbeat in the CI (elastic#22252)
  Fix awscloudwatch input documentation (elastic#22247)
  Add support for different Azure Cloud environments in the metricbeat azure module (elastic#21044)
  [CI] support windows-2008-r2 (elastic#19791)
  protect against accessing undefined variables in sysmon module (elastic#22236)
  [CI] archive only if failed steps (elastic#22220)
  Add pe fields to Sysmon module (elastic#22217)
  [CI][flaky] Support 7.x branches and PRs (elastic#22197)
  Perfmon - Fix regular expressions to comply to multiple parentheses in instance name and object (elastic#22146)
  ci: improve linting speed (elastic#22103)
  Move cloudfoundry tags with metadata to common metadata fields (elastic#22150)
  [Docs] Update custom beat docs (elastic#22194)
  [Ingest Manager] Agent fix snapshot download for upgrade (elastic#22175)
  Update shared-autodiscover.asciidoc (elastic#21827)
  [DOCS] Warn about compression and Azure Event Hub for Kafka (elastic#21578)
  [CI][flaky] reporting for PRs in GitHub (elastic#21853)
  [Packetbeat] Create x-pack magefile (elastic#21979)
  [Elastic Agent] Fix deb/rpm installation (elastic#22153)
  ...
  • Loading branch information
@v1v
v1v committed Oct 29, 2020
2 parents 1fe3a2c + 633285e commit 3f9b862
Show file tree
Hide file tree
Showing 221 changed files with 7,739 additions and 1,259 deletions.
3 changes: 3 additions & 0 deletions .ci/beats-tester.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,15 @@ pipeline {
options { skipDefaultCheckout() }
when { branch 'master' }
steps {
// TODO: to use the git commit that triggered the upstream build
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
}
}
stage('Build *.x branch') {
options { skipDefaultCheckout() }
when { branch '*.x' }
steps {
// TODO: to use the git commit that triggered the upstream build
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
}
}
Expand All @@ -84,6 +86,7 @@ pipeline {
}
}
steps {
// TODO: to use the git commit that triggered the upstream build
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
}
}
Expand Down
159 changes: 103 additions & 56 deletions .ci/packaging.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@

@Library('apm@current') _

import groovy.transform.Field

/**
This is required to store the test suites we will use to trigger the E2E tests.
*/
@Field def e2eTestSuites = []

pipeline {
agent none
environment {
Expand Down Expand Up @@ -36,7 +43,7 @@ pipeline {
}
stages {
stage('Filter build') {
agent { label 'ubuntu && immutable' }
agent { label 'ubuntu-18 && immutable' }
when {
beforeAgent true
anyOf {
Expand Down Expand Up @@ -84,14 +91,14 @@ pipeline {
'x-pack/heartbeat',
// 'x-pack/journalbeat',
'x-pack/metricbeat',
// 'x-pack/packetbeat',
'x-pack/packetbeat',
'x-pack/winlogbeat'
)
}
}
stages {
stage('Package Linux'){
agent { label 'ubuntu && immutable' }
agent { label 'ubuntu-18 && immutable' }
options { skipDefaultCheckout() }
when {
beforeAgent true
Expand Down Expand Up @@ -121,7 +128,7 @@ pipeline {
release()
pushCIDockerImages()
}
runE2ETestForPackages()
prepareE2ETestForPackage("${BEATS_FOLDER}")
}
}
stage('Package Mac OS'){
Expand Down Expand Up @@ -152,45 +159,46 @@ pipeline {
}
}
}
stage('Run E2E Tests for Packages'){
agent { label 'ubuntu-18 && immutable' }
options { skipDefaultCheckout() }
steps {
runE2ETests()
}
}
}
}
}
}

def pushCIDockerImages(){
catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
if ("${env.BEATS_FOLDER}" == "auditbeat"){
tagAndPush('auditbeat-oss')
} else if ("${env.BEATS_FOLDER}" == "filebeat") {
tagAndPush('filebeat-oss')
} else if ("${env.BEATS_FOLDER}" == "heartbeat"){
tagAndPush('heartbeat-oss')
if (env?.BEATS_FOLDER?.endsWith('auditbeat')) {
tagAndPush('auditbeat')
} else if (env?.BEATS_FOLDER?.endsWith('filebeat')) {
tagAndPush('filebeat')
} else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) {
tagAndPush('heartbeat')
} else if ("${env.BEATS_FOLDER}" == "journalbeat"){
tagAndPush('journalbeat')
tagAndPush('journalbeat-oss')
} else if ("${env.BEATS_FOLDER}" == "metricbeat"){
tagAndPush('metricbeat-oss')
} else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
tagAndPush('metricbeat')
} else if ("${env.BEATS_FOLDER}" == "packetbeat"){
tagAndPush('packetbeat')
tagAndPush('packetbeat-oss')
} else if ("${env.BEATS_FOLDER}" == "x-pack/auditbeat"){
tagAndPush('auditbeat')
} else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
tagAndPush('elastic-agent')
} else if ("${env.BEATS_FOLDER}" == "x-pack/filebeat"){
tagAndPush('filebeat')
} else if ("${env.BEATS_FOLDER}" == "x-pack/heartbeat"){
tagAndPush('heartbeat')
} else if ("${env.BEATS_FOLDER}" == "x-pack/metricbeat"){
tagAndPush('metricbeat')
}
}
}

def tagAndPush(name){
def tagAndPush(beatName){
def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
def aliasVersion = ""
if("${env.SNAPSHOT}" == "true"){
aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version

libbetaVer += "-SNAPSHOT"
aliasVersion += "-SNAPSHOT"
}

def tagName = "${libbetaVer}"
Expand All @@ -203,44 +211,53 @@ def tagAndPush(name){
// supported image flavours
def variants = ["", "-oss", "-ubi8"]
variants.each { variant ->
def oldName = "${DOCKER_REGISTRY}/beats/${name}${variant}:${libbetaVer}"
def newName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${tagName}"
def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${env.GIT_BASE_COMMIT}"

def iterations = 0
retryWithSleep(retries: 3, seconds: 5, backoff: true)
iterations++
def status = sh(label:'Change tag and push', script: """
docker tag ${oldName} ${newName}
docker push ${newName}
docker tag ${oldName} ${commitName}
docker push ${commitName}
""", returnStatus: true)
doTagAndPush(beatName, variant, libbetaVer, tagName)
doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")

if ( status > 0 && iterations < 3) {
error('tag and push failed, retry')
} else if ( status > 0 ) {
log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
if (!isPR() && aliasVersion != "") {
doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
}
}
}

def runE2ETestForPackages(){
def suite = ''
/**
* @param beatName name of the Beat
* @param variant name of the variant used to build the docker image name
* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
*/
def doTagAndPush(beatName, variant, sourceTag, targetTag) {
def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"

catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') {
if ("${env.BEATS_FOLDER}" == "filebeat" || "${env.BEATS_FOLDER}" == "x-pack/filebeat") {
suite = 'helm,fleet'
} else if ("${env.BEATS_FOLDER}" == "metricbeat" || "${env.BEATS_FOLDER}" == "x-pack/metricbeat") {
suite = ''
} else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
suite = 'fleet'
} else {
echo("Skipping E2E tests for ${env.BEATS_FOLDER}.")
return
def iterations = 0
retryWithSleep(retries: 3, seconds: 5, backoff: true) {
iterations++
def status = sh(label: "Change tag and push ${targetName}", script: """
docker tag ${sourceName} ${targetName}
docker push ${targetName}
""", returnStatus: true)

if ( status > 0 && iterations < 3) {
error("tag and push failed for ${beatName}, retry")
} else if ( status > 0 ) {
log(level: 'WARN', text: "${beatName} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
}
}
}

triggerE2ETests(suite)
def prepareE2ETestForPackage(String beat){
if ("${beat}" == "filebeat" || "${beat}" == "x-pack/filebeat") {
e2eTestSuites.push('fleet')
e2eTestSuites.push('helm')
} else if ("${beat}" == "metricbeat" || "${beat}" == "x-pack/metricbeat") {
e2eTestSuites.push('ALL')
echo("${beat} adds all test suites to the E2E tests job.")
} else if ("${beat}" == "x-pack/elastic-agent") {
e2eTestSuites.push('fleet')
} else {
echo("${beat} does not add any test suite to the E2E tests job.")
return
}
}

Expand All @@ -257,8 +274,29 @@ def release(){
}
}

def runE2ETests(){
if (e2eTestSuites.size() == 0) {
echo("Not triggering E2E tests for PR-${env.CHANGE_ID} because the changes does not affect the E2E.")
return
}

def suites = '' // empty value represents all suites in the E2E tests

catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') {
def suitesSet = e2eTestSuites.toSet()

if (!suitesSet.contains('ALL')) {
suitesSet.each { suite ->
suites += "${suite},"
};
}

triggerE2ETests(suites)
}
}

def triggerE2ETests(String suite) {
echo("Triggering E2E tests for ${env.BEATS_FOLDER}. Test suite: ${suite}.")
echo("Triggering E2E tests for PR-${env.CHANGE_ID}. Test suites: ${suite}.")

def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
Expand All @@ -285,7 +323,7 @@ def triggerE2ETests(String suite) {
wait: false
)

def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME} for ${env.BEATS_FOLDER}"
def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME}"
githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
}

Expand All @@ -305,7 +343,16 @@ def publishPackages(baseDir){
bucketUri = "gs://${JOB_GCS_BUCKET}/pull-requests/pr-${env.CHANGE_ID}"
}
def beatsFolderName = getBeatsName(baseDir)
googleStorageUpload(bucket: "${bucketUri}/${beatsFolderName}",
uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)

// Copy those files to another location with the sha commit to test them
// aftewords.
bucketUri = "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}"
uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
}

def uploadPackages(bucketUri, baseDir){
googleStorageUpload(bucket: bucketUri,
credentialsId: "${JOB_GCS_CREDENTIALS}",
pathPrefix: "${baseDir}/build/distributions/",
pattern: "${baseDir}/build/distributions/**/*",
Expand Down
Loading

0 comments on commit 3f9b862

Please sign in to comment.