Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: stage execution cache #24780

Merged
merged 11 commits into from
Apr 5, 2021
Merged

feat: stage execution cache #24780

Changes from 6 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
a762fbf
feat: stage execution cache
kuisathaverat Mar 26, 2021
b56978e
fix: use correct context
kuisathaverat Mar 26, 2021
1ecf54a
fix: do not check stage status on the first run
kuisathaverat Mar 26, 2021
dc2e1b4
fix: proper URL
kuisathaverat Mar 26, 2021
755661f
chore: show message when the stache is skip
kuisathaverat Mar 26, 2021
a00d154
fix: correct path
kuisathaverat Mar 26, 2021
400e83d
fix: add final /
kuisathaverat Mar 29, 2021
d110fbe
test: is the path needed?
kuisathaverat Mar 29, 2021
bd4747b
fix: remove prefix
kuisathaverat Mar 29, 2021
ddfe503
chore: refactor to use curl to download
kuisathaverat Mar 29, 2021
06bde94
chore: use pipeline step
kuisathaverat Mar 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 92 additions & 40 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,18 +73,20 @@ pipeline {
GOFLAGS = '-mod=readonly'
}
steps {
withGithubNotify(context: "Lint") {
withBeatsEnv(archive: false, id: "lint") {
dumpVariables()
setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
whenTrue(env.ONLY_DOCS == 'true') {
cmd(label: "make check", script: "make check")
}
whenTrue(env.ONLY_DOCS == 'false') {
cmd(label: "make check-python", script: "make check-python")
cmd(label: "make check-go", script: "make check-go")
cmd(label: "make notice", script: "make notice")
cmd(label: "Check for changes", script: "make check-no-changes")
stageStatusCache(id: 'Lint'){
withGithubNotify(context: "Lint") {
withBeatsEnv(archive: false, id: "lint") {
dumpVariables()
setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
whenTrue(env.ONLY_DOCS == 'true') {
cmd(label: "make check", script: "make check")
}
whenTrue(env.ONLY_DOCS == 'false') {
cmd(label: "make check-python", script: "make check-python")
cmd(label: "make check-go", script: "make check-go")
cmd(label: "make notice", script: "make notice")
cmd(label: "Check for changes", script: "make check-no-changes")
}
}
}
}
Expand Down Expand Up @@ -959,40 +961,42 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
super(args)
}
public run(Map args = [:]){
def withModule = args.content.get('withModule', false)
if(args?.content?.containsKey('make')) {
steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id)
}
if(args?.content?.containsKey('mage')) {
steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
}
if(args?.content?.containsKey('packaging-arm')) {
steps.packagingArm(context: args.context,
command: args.content.get('packaging-arm'),
directory: args.project,
label: args.label,
isMage: true,
id: args.id,
e2e: args.content.get('e2e'),
package: true,
dockerArch: 'arm64')
}
if(args?.content?.containsKey('packaging-linux')) {
steps.packagingLinux(context: args.context,
command: args.content.get('packaging-linux'),
steps.stageStatusCache(args){
def withModule = args.content.get('withModule', false)
if(args?.content?.containsKey('make')) {
steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id)
}
if(args?.content?.containsKey('mage')) {
steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
}
if(args?.content?.containsKey('packaging-arm')) {
steps.packagingArm(context: args.context,
command: args.content.get('packaging-arm'),
directory: args.project,
label: args.label,
isMage: true,
id: args.id,
e2e: args.content.get('e2e'),
package: true,
dockerArch: 'amd64')
}
if(args?.content?.containsKey('k8sTest')) {
steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
}
if(args?.content?.containsKey('cloud')) {
steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id)
dockerArch: 'arm64')
}
if(args?.content?.containsKey('packaging-linux')) {
steps.packagingLinux(context: args.context,
command: args.content.get('packaging-linux'),
directory: args.project,
label: args.label,
isMage: true,
id: args.id,
e2e: args.content.get('e2e'),
package: true,
dockerArch: 'amd64')
}
if(args?.content?.containsKey('k8sTest')) {
steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
}
if(args?.content?.containsKey('cloud')) {
steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id)
}
}
}
}
Expand All @@ -1014,3 +1018,51 @@ class GetProjectDependencies extends co.elastic.beats.BeatsFunction {
return output?.split('\n').collect{ item -> item as String }
}
}

/**
Execute the body if there is not stage status file for the stage and the commit we are building.
User triggered builds will execute all stages always.
If the stage success the status is save in a file.
*/
def stageStatusCache(Map args, Closure body){
if(readStageStatus(args) == false || isUserTrigger() || env.BUILD_ID == "1"){
body()
saveStageStatus(args)
} else {
log(level: 'INFO', text: "The stage skiped because it is in the execution cache.")
}
}

/**
Save the status file of the stage.
*/
def saveStageStatus(Map args){
def statusFileName = stageStatusId(args)
writeFile(file: statusFileName, text: "OK")
googleStorageUploadExt(bucket: "gs://${JOB_GCS_BUCKET}/ci/cache",
credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
pattern: "${statusFileName}",
sharedPublicly: true)
}

/**
Read the status file of the stage if it exists.
*/
def readStageStatus(Map args){
def statusFileName = stageStatusId(args)
try {
googleStorageDownload(bucketUri: "gs://${JOB_GCS_BUCKET}/ci/cache/${statusFileName}",
credentialsId: "${JOB_GCS_CREDENTIALS}")
} catch(e) {
log(level: 'WARN', text: "There is no cache file for the current stage.")
} finally {
return fileExists("${statusFileName}")
}
}

/**
generate an unique ID for the stage and commit.
*/
def stageStatusId(Map args){
return base64encode(text: "${args.id}${GIT_BASE_COMMIT}", encoding: "UTF-8")
}