From a762fbfb8b1fc18e9d1fc55767666acfa82ddb14 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 26 Mar 2021 11:59:40 +0100 Subject: [PATCH 01/11] feat: stage execution cache --- Jenkinsfile | 130 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 40 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c910919c954..35c38d27bd4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,18 +73,20 @@ pipeline { GOFLAGS = '-mod=readonly' } steps { - withGithubNotify(context: "Lint") { - withBeatsEnv(archive: false, id: "lint") { - dumpVariables() - setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) - whenTrue(env.ONLY_DOCS == 'true') { - cmd(label: "make check", script: "make check") - } - whenTrue(env.ONLY_DOCS == 'false') { - cmd(label: "make check-python", script: "make check-python") - cmd(label: "make check-go", script: "make check-go") - cmd(label: "make notice", script: "make notice") - cmd(label: "Check for changes", script: "make check-no-changes") + stageStatusCache(id: 'Lint'){ + withGithubNotify(context: "Lint") { + withBeatsEnv(archive: false, id: "lint") { + dumpVariables() + setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) + whenTrue(env.ONLY_DOCS == 'true') { + cmd(label: "make check", script: "make check") + } + whenTrue(env.ONLY_DOCS == 'false') { + cmd(label: "make check-python", script: "make check-python") + cmd(label: "make check-go", script: "make check-go") + cmd(label: "make notice", script: "make notice") + cmd(label: "Check for changes", script: "make check-no-changes") + } } } } @@ -959,40 +961,42 @@ class RunCommand extends co.elastic.beats.BeatsFunction { super(args) } public run(Map args = [:]){ - def withModule = args.content.get('withModule', false) - if(args?.content?.containsKey('make')) { - steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) - } - if(args?.content?.containsKey('mage')) { - steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) - } - if(args?.content?.containsKey('packaging-arm')) { - steps.packagingArm(context: args.context, - command: args.content.get('packaging-arm'), - directory: args.project, - label: args.label, - isMage: true, - id: args.id, - e2e: args.content.get('e2e'), - package: true, - dockerArch: 'arm64') - } - if(args?.content?.containsKey('packaging-linux')) { - steps.packagingLinux(context: args.context, - command: args.content.get('packaging-linux'), + stageStatusCache(args){ + def withModule = args.content.get('withModule', false) + if(args?.content?.containsKey('make')) { + steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) + } + if(args?.content?.containsKey('mage')) { + steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) + } + if(args?.content?.containsKey('packaging-arm')) { + steps.packagingArm(context: args.context, + command: args.content.get('packaging-arm'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'), package: true, - dockerArch: 'amd64') - } - if(args?.content?.containsKey('k8sTest')) { - steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) - } - if(args?.content?.containsKey('cloud')) { - steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) + dockerArch: 'arm64') + } + if(args?.content?.containsKey('packaging-linux')) { + steps.packagingLinux(context: args.context, + command: args.content.get('packaging-linux'), + directory: args.project, + label: args.label, + isMage: true, + id: args.id, + e2e: args.content.get('e2e'), + package: true, + dockerArch: 'amd64') + } + if(args?.content?.containsKey('k8sTest')) { + steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) + } + if(args?.content?.containsKey('cloud')) { + steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) + } } } } @@ -1014,3 +1018,49 @@ class GetProjectDependencies extends co.elastic.beats.BeatsFunction { return output?.split('\n').collect{ item -> item as String } } } + +/** + Execute the body if there is not stage status file for the stage and the commit we are building. + User triggered builds will execute all stages always. + If the stage success the status is save in a file. +*/ +def stageStatusCache(Map args, Closure body){ + if(readStageStatus(args) == false || isUserTrigger()){ + body() + saveStageStatus(args) + } +} + +/** + Save the status file of the stage. +*/ +def saveStageStatus(Map args){ + def statusFileName = stageStatusId(args) + writeFile(file: statusFileName, text: "OK") + googleStorageUploadExt(bucket: "${JOB_GCS_BUCKET}/ci/cache", + credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", + pattern: "${statusFileName}", + sharedPublicly: true) +} + +/** + Read the status file of the stage if it exists. +*/ +def readStageStatus(Map args){ + def statusFileName = stageStatusId(args) + try { + googleStorageDownload(bucketUri: "${JOB_GCS_BUCKET}/ci/cache/${statusFileName}", + credentialsId: "${JOB_GCS_CREDENTIALS}") + } catch(e) { + log(level: 'WARN', text: "There is no cache file for the current stage.") + } finally { + return fileExists("${statusFileName}") + } +} + +/** + generate an unique ID for the stage and commit. +*/ +def stageStatusId(Map args){ + return base64encode(text: "${args.id}${GIT_BASE_COMMIT}", encoding: "UTF-8") +} From b56978e7bf527440a263d72f8e5866155c7f465c Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 26 Mar 2021 12:29:50 +0100 Subject: [PATCH 02/11] fix: use correct context --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 35c38d27bd4..969e56e07aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -961,7 +961,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction { super(args) } public run(Map args = [:]){ - stageStatusCache(args){ + steps.stageStatusCache(args){ def withModule = args.content.get('withModule', false) if(args?.content?.containsKey('make')) { steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) From 1ecf54a8ab54b0fcc12cff583543db6b90eed910 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 26 Mar 2021 15:43:01 +0100 Subject: [PATCH 03/11] fix: do not check stage status on the first run --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 969e56e07aa..cdb3ce7ac5b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1025,7 +1025,7 @@ class GetProjectDependencies extends co.elastic.beats.BeatsFunction { If the stage success the status is save in a file. */ def stageStatusCache(Map args, Closure body){ - if(readStageStatus(args) == false || isUserTrigger()){ + if(readStageStatus(args) == false || isUserTrigger() || env.BUILD_ID == "1"){ body() saveStageStatus(args) } From dc2e1b4dc007e0786a22e74e00cf6faf71a60906 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 26 Mar 2021 15:43:18 +0100 Subject: [PATCH 04/11] fix: proper URL --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index cdb3ce7ac5b..c944edb4a1f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1049,7 +1049,7 @@ def saveStageStatus(Map args){ def readStageStatus(Map args){ def statusFileName = stageStatusId(args) try { - googleStorageDownload(bucketUri: "${JOB_GCS_BUCKET}/ci/cache/${statusFileName}", + googleStorageDownload(bucketUri: "gs://${JOB_GCS_BUCKET}/ci/cache/${statusFileName}", credentialsId: "${JOB_GCS_CREDENTIALS}") } catch(e) { log(level: 'WARN', text: "There is no cache file for the current stage.") From 755661fdb6eb09b33701e27592fe1da4883c8c2c Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 26 Mar 2021 17:32:35 +0100 Subject: [PATCH 05/11] chore: show message when the stache is skip --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index c944edb4a1f..6fa635a9d33 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1028,6 +1028,8 @@ def stageStatusCache(Map args, Closure body){ if(readStageStatus(args) == false || isUserTrigger() || env.BUILD_ID == "1"){ body() saveStageStatus(args) + } else { + log(level: 'INFO', text: "The stage skiped because it is in the execution cache.") } } From a00d154b07a5517fcaf183411d0456902556b958 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 26 Mar 2021 17:32:48 +0100 Subject: [PATCH 06/11] fix: correct path --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6fa635a9d33..2f9584710a7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1039,7 +1039,7 @@ def stageStatusCache(Map args, Closure body){ def saveStageStatus(Map args){ def statusFileName = stageStatusId(args) writeFile(file: statusFileName, text: "OK") - googleStorageUploadExt(bucket: "${JOB_GCS_BUCKET}/ci/cache", + googleStorageUploadExt(bucket: "gs://${JOB_GCS_BUCKET}/ci/cache", credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", pattern: "${statusFileName}", sharedPublicly: true) From 400e83d7427f28b5b7b0b104e9eba44f5c59b7f4 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Mon, 29 Mar 2021 10:07:52 +0200 Subject: [PATCH 07/11] fix: add final / --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2f9584710a7..0edefec7eb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1039,7 +1039,7 @@ def stageStatusCache(Map args, Closure body){ def saveStageStatus(Map args){ def statusFileName = stageStatusId(args) writeFile(file: statusFileName, text: "OK") - googleStorageUploadExt(bucket: "gs://${JOB_GCS_BUCKET}/ci/cache", + googleStorageUploadExt(bucket: "gs://${JOB_GCS_BUCKET}/ci/cache/", credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", pattern: "${statusFileName}", sharedPublicly: true) From d110fbec2e531b95a803163eed2db5f87d063218 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Mon, 29 Mar 2021 11:08:44 +0200 Subject: [PATCH 08/11] test: is the path needed? --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0edefec7eb9..4ebb53108b3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1052,7 +1052,8 @@ def readStageStatus(Map args){ def statusFileName = stageStatusId(args) try { googleStorageDownload(bucketUri: "gs://${JOB_GCS_BUCKET}/ci/cache/${statusFileName}", - credentialsId: "${JOB_GCS_CREDENTIALS}") + credentialsId: "${JOB_GCS_CREDENTIALS}", + localDirectory: '.') } catch(e) { log(level: 'WARN', text: "There is no cache file for the current stage.") } finally { From bd4747bd7bad2e67af32b127021207fe0105c39d Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Mon, 29 Mar 2021 12:58:41 +0200 Subject: [PATCH 09/11] fix: remove prefix --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4ebb53108b3..6b9b838122b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1053,7 +1053,8 @@ def readStageStatus(Map args){ try { googleStorageDownload(bucketUri: "gs://${JOB_GCS_BUCKET}/ci/cache/${statusFileName}", credentialsId: "${JOB_GCS_CREDENTIALS}", - localDirectory: '.') + localDirectory: '.', + pathPrefix: 'ci/cache/') } catch(e) { log(level: 'WARN', text: "There is no cache file for the current stage.") } finally { From ddfe503943a7435eb387009d7c7cb41b0cbb7188 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Mon, 29 Mar 2021 17:48:34 +0200 Subject: [PATCH 10/11] chore: refactor to use curl to download --- Jenkinsfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6b9b838122b..230a24b07f2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1051,10 +1051,9 @@ def saveStageStatus(Map args){ def readStageStatus(Map args){ def statusFileName = stageStatusId(args) try { - googleStorageDownload(bucketUri: "gs://${JOB_GCS_BUCKET}/ci/cache/${statusFileName}", - credentialsId: "${JOB_GCS_CREDENTIALS}", - localDirectory: '.', - pathPrefix: 'ci/cache/') + cmd(label: 'Download Stage Status', + script: "curl -sSf -O https://storage.googleapis.com/${JOB_GCS_BUCKET}/ci/cache/${statusFileName}", + returnStatus: true) } catch(e) { log(level: 'WARN', text: "There is no cache file for the current stage.") } finally { From 06bde946389d35953f73323b6d6a55d1faea9596 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 31 Mar 2021 13:25:14 +0200 Subject: [PATCH 11/11] chore: use pipeline step --- Jenkinsfile | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 230a24b07f2..7fe3ce62904 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1018,52 +1018,3 @@ class GetProjectDependencies extends co.elastic.beats.BeatsFunction { return output?.split('\n').collect{ item -> item as String } } } - -/** - Execute the body if there is not stage status file for the stage and the commit we are building. - User triggered builds will execute all stages always. - If the stage success the status is save in a file. -*/ -def stageStatusCache(Map args, Closure body){ - if(readStageStatus(args) == false || isUserTrigger() || env.BUILD_ID == "1"){ - body() - saveStageStatus(args) - } else { - log(level: 'INFO', text: "The stage skiped because it is in the execution cache.") - } -} - -/** - Save the status file of the stage. -*/ -def saveStageStatus(Map args){ - def statusFileName = stageStatusId(args) - writeFile(file: statusFileName, text: "OK") - googleStorageUploadExt(bucket: "gs://${JOB_GCS_BUCKET}/ci/cache/", - credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", - pattern: "${statusFileName}", - sharedPublicly: true) -} - -/** - Read the status file of the stage if it exists. -*/ -def readStageStatus(Map args){ - def statusFileName = stageStatusId(args) - try { - cmd(label: 'Download Stage Status', - script: "curl -sSf -O https://storage.googleapis.com/${JOB_GCS_BUCKET}/ci/cache/${statusFileName}", - returnStatus: true) - } catch(e) { - log(level: 'WARN', text: "There is no cache file for the current stage.") - } finally { - return fileExists("${statusFileName}") - } -} - -/** - generate an unique ID for the stage and commit. -*/ -def stageStatusId(Map args){ - return base64encode(text: "${args.id}${GIT_BASE_COMMIT}", encoding: "UTF-8") -}