Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/7.5' into backport/7.5/pr-52834
Browse files Browse the repository at this point in the history
  • Loading branch information
Aaron Caldwell committed Jan 7, 2020
2 parents 2c9020a + 25d4619 commit 452c4f5
Show file tree
Hide file tree
Showing 24 changed files with 832 additions and 224 deletions.
162 changes: 162 additions & 0 deletions .ci/es-snapshots/Jenkinsfile_build_es
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
#!/bin/groovy

// This job effectively has two SCM configurations:
// one for kibana, used to check out this Jenkinsfile (which means it's the job's main SCM configuration), as well as kick-off the downstream verification job
// one for elasticsearch, used to check out the elasticsearch source before building it

// There are two parameters that drive which branch is checked out for each of these, but they will typically be the same
// 'branch_specifier' is for kibana / the job itself
// ES_BRANCH is for elasticsearch

library 'kibana-pipeline-library'
kibanaLibrary.load()

def ES_BRANCH = params.ES_BRANCH

if (!ES_BRANCH) {
error "Parameter 'ES_BRANCH' must be specified."
}

currentBuild.displayName += " - ${ES_BRANCH}"
currentBuild.description = "ES: ${ES_BRANCH}<br />Kibana: ${params.branch_specifier}"

def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION

timeout(time: 120, unit: 'MINUTES') {
timestamps {
ansiColor('xterm') {
node('linux && immutable') {
catchError {
def VERSION
def SNAPSHOT_ID
def DESTINATION

def scmVars = checkoutEs(ES_BRANCH)
def GIT_COMMIT = scmVars.GIT_COMMIT
def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim()

buildArchives('to-archive')

dir('to-archive') {
def now = new Date()
def date = now.format("yyyyMMdd-HHmmss")

def filesRaw = sh(script: "ls -1", returnStdout: true).trim()
def files = filesRaw
.split("\n")
.collect { filename ->
// Filename examples
// elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
// elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
def parts = filename.replace("elasticsearch-oss", "oss").split("-")

VERSION = VERSION ?: parts[1]
SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}"
DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}"

return [
filename: filename,
checksum: filename + '.sha512',
url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(),
version: parts[1],
platform: parts[3],
architecture: parts[4].split('\\.')[0],
license: parts[0] == 'oss' ? 'oss' : 'default',
]
}

sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;'

def manifest = [
bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(),
branch: ES_BRANCH,
sha: GIT_COMMIT,
sha_short: GIT_COMMIT_SHORT,
version: VERSION,
generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")),
archives: files,
]
def manifestJson = toJSON(manifest).toString()
writeFile file: 'manifest.json', text: manifestJson

upload(DESTINATION, '*.*')

sh "cp manifest.json manifest-latest.json"
upload(VERSION, 'manifest-latest.json')
}

if (PROMOTE_WITHOUT_VERIFY) {
esSnapshots.promote(VERSION, SNAPSHOT_ID)

emailext(
to: 'build-kibana@elastic.co',
subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}",
body: '${SCRIPT,template="groovy-html.template"}',
mimeType: 'text/html',
)
} else {
build(
propagate: false,
wait: false,
job: 'elasticsearch+snapshots+verify',
parameters: [
string(name: 'branch_specifier', value: branch_specifier),
string(name: 'SNAPSHOT_VERSION', value: VERSION),
string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID),
]
)
}
}

kibanaPipeline.sendMail()
}
}
}
}

def checkoutEs(branch) {
retryWithDelay(8, 15) {
return checkout([
$class: 'GitSCM',
branches: [[name: branch]],
doGenerateSubmoduleConfigurations: false,
extensions: [],
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
url: 'git@github.com:elastic/elasticsearch',
]],
])
}
}

def upload(destination, pattern) {
return googleStorageUpload(
credentialsId: 'kibana-ci-gcs-plugin',
bucket: "gs://kibana-ci-es-snapshots-daily/${destination}",
pattern: pattern,
sharedPublicly: false,
showInline: false,
)
}

def buildArchives(destination) {
def props = readProperties file: '.ci/java-versions.properties'
withEnv([
// Select the correct JDK for this branch
"PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}",

// These Jenkins env vars trigger some automation in the elasticsearch repo that we don't want
"BUILD_NUMBER=",
"JENKINS_URL=",
"BUILD_URL=",
"JOB_NAME=",
"NODE_NAME=",
]) {
sh """
./gradlew -p distribution/archives assemble --parallel
mkdir -p ${destination}
find distribution/archives -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\;
"""
}
}
19 changes: 19 additions & 0 deletions .ci/es-snapshots/Jenkinsfile_trigger_build_es
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/bin/groovy

if (!params.branches_yaml) {
error "'branches_yaml' parameter must be specified"
}

def branches = readYaml text: params.branches_yaml

branches.each { branch ->
build(
propagate: false,
wait: false,
job: 'elasticsearch+snapshots+build',
parameters: [
string(name: 'branch_specifier', value: branch),
string(name: 'ES_BRANCH', value: branch),
]
)
}
72 changes: 72 additions & 0 deletions .ci/es-snapshots/Jenkinsfile_verify_es
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!/bin/groovy

library 'kibana-pipeline-library'
kibanaLibrary.load()

def SNAPSHOT_VERSION = params.SNAPSHOT_VERSION
def SNAPSHOT_ID = params.SNAPSHOT_ID

if (!SNAPSHOT_VERSION) {
error "Parameter SNAPSHOT_VERSION must be specified"
}

if (!SNAPSHOT_ID) {
error "Parameter SNAPSHOT_ID must be specified"
}

currentBuild.displayName += " - ${SNAPSHOT_VERSION}"
currentBuild.description = "ES: ${SNAPSHOT_VERSION}<br />Kibana: ${params.branch_specifier}"

def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json"

timeout(time: 120, unit: 'MINUTES') {
timestamps {
ansiColor('xterm') {
catchError {
withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
parallel([
// TODO we just need to run integration tests from intake?
'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'),
'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'),
'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4),
'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5),
'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6),
'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7),
'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8),
'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9),
'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
]),
'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
]),
])
}

promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID)
}

kibanaPipeline.sendMail()
}
}
}

def promoteSnapshot(snapshotVersion, snapshotId) {
node('linux && immutable') {
esSnapshots.promote(snapshotVersion, snapshotId)
}
}
Loading

0 comments on commit 452c4f5

Please sign in to comment.