Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[CI] fix MODULE variable cornercases #19985

Merged
merged 1 commit into from
Jul 17, 2020
Merged

[CI] fix MODULE variable cornercases #19985

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 36 additions & 28 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ pipeline {
}
}
steps {
makeTarget(context: "Filebeat oss Linux", target: "-C filebeat testsuite", withModule: true)
makeTarget(context: "Filebeat oss Linux", directory: 'filebeat', target: 'testsuite', withModule: true)
}
}
stage('Filebeat x-pack'){
Expand Down Expand Up @@ -243,7 +243,7 @@ pipeline {
stages {
stage('Heartbeat oss'){
steps {
makeTarget(context: "Heartbeat oss Linux", target: "-C heartbeat testsuite")
makeTarget(context: "Heartbeat oss Linux", directory: 'heartbeat', target: "testsuite")
}
}
stage('Heartbeat Mac OS X'){
Expand Down Expand Up @@ -289,7 +289,7 @@ pipeline {
}
}
steps {
makeTarget(context: "Auditbeat oss Linux", target: "-C auditbeat testsuite", withModule: true)
makeTarget(context: "Auditbeat oss Linux", directory: 'auditbeat', target: "testsuite", withModule: true)
}
}
stage('Auditbeat crosscompile'){
Expand All @@ -302,7 +302,7 @@ pipeline {
}
}
steps {
makeTarget(context: "Auditbeat oss crosscompile", target: "-C auditbeat crosscompile")
makeTarget(context: "Auditbeat oss crosscompile", directory: 'auditbeat', target: "crosscompile")
}
}
stage('Auditbeat oss Mac OS X'){
Expand Down Expand Up @@ -387,12 +387,12 @@ pipeline {
stages {
stage('Libbeat oss'){
steps {
makeTarget(context: "Libbeat oss Linux", target: "-C libbeat testsuite")
makeTarget(context: "Libbeat oss Linux", directory: 'libbeat', target: "testsuite")
}
}
stage('Libbeat crosscompile'){
steps {
makeTarget(context: "Libbeat oss crosscompile", target: "-C libbeat crosscompile")
makeTarget(context: "Libbeat oss crosscompile", directory: 'libbeat', target: "crosscompile")
}
}
stage('Libbeat stress-tests'){
Expand All @@ -412,7 +412,7 @@ pipeline {
}
}
steps {
makeTarget(context: "Libbeat x-pack Linux", target: "-C x-pack/libbeat testsuite")
makeTarget(context: "Libbeat x-pack Linux", directory: 'x-pack/libbeat', target: "testsuite")
}
}
stage('Metricbeat OSS Unit tests'){
Expand Down Expand Up @@ -499,7 +499,7 @@ pipeline {
}
}
steps {
makeTarget(context: "Metricbeat OSS crosscompile", target: "-C metricbeat crosscompile")
makeTarget(context: "Metricbeat OSS crosscompile", directory: 'metricbeat', target: "crosscompile")
}
}
stage('Metricbeat Mac OS X'){
Expand Down Expand Up @@ -571,7 +571,7 @@ pipeline {
stages {
stage('Packetbeat oss'){
steps {
makeTarget(context: "Packetbeat oss Linux", target: "-C packetbeat testsuite")
makeTarget(context: "Packetbeat oss Linux", directory: 'packetbeat', target: "testsuite")
}
}
}
Expand Down Expand Up @@ -605,7 +605,7 @@ pipeline {
stages {
stage('Winlogbeat oss'){
steps {
makeTarget(context: "Winlogbeat oss crosscompile", target: "-C winlogbeat crosscompile")
makeTarget(context: "Winlogbeat oss crosscompile", directory: 'winlogbeat', target: "crosscompile")
}
}
stage('Winlogbeat Windows'){
Expand Down Expand Up @@ -650,7 +650,7 @@ pipeline {
steps {
mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "update build test")
withEnv(["GO_VERSION=1.13.1"]){
makeTarget(context: "Functionbeat x-pack Linux", target: "-C x-pack/functionbeat test-gcp-functions")
makeTarget(context: "Functionbeat x-pack Linux", directory: 'x-pack/functionbeat', target: "test-gcp-functions")
}
}
}
Expand Down Expand Up @@ -699,7 +699,7 @@ pipeline {
stages {
stage('Journalbeat oss'){
steps {
makeTarget(context: "Journalbeat Linux", target: "-C journalbeat testsuite")
makeTarget(context: "Journalbeat Linux", directory: 'journalbeat', target: "testsuite")
}
}
}
Expand All @@ -716,14 +716,14 @@ pipeline {
stages {
stage('Generators Metricbeat Linux'){
steps {
makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test")
makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test-package")
makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test")
makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test-package")
}
}
stage('Generators Beat Linux'){
steps {
makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test")
makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test-package")
makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test")
makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test-package")
}
}
stage('Generators Metricbeat Mac OS X'){
Expand All @@ -736,7 +736,7 @@ pipeline {
}
}
steps {
makeTarget(context: "Generators Metricbeat Mac OS X", target: "-C generator/_templates/metricbeat test")
makeTarget(context: "Generators Metricbeat Mac OS X", directory: 'generator/_templates/metricbeat', target: "test")
}
post {
always {
Expand All @@ -754,7 +754,7 @@ pipeline {
}
}
steps {
makeTarget(context: "Generators Beat Mac OS X", target: "-C generator/_templates/beat test")
makeTarget(context: "Generators Beat Mac OS X", directory: 'generator/_templates/beat', target: "test")
}
post {
always {
Expand Down Expand Up @@ -807,15 +807,17 @@ def fixPermissions(location) {
def makeTarget(Map args = [:]) {
def context = args.context
def target = args.target
def directory = args.get('directory', '')
def clean = args.get('clean', true)
def withModule = args.get('withModule', false)
def directoryFlag = directory.trim() ? "-C ${directory}" : ''
withGithubNotify(context: "${context}") {
withBeatsEnv(archive: true, withModule: withModule, modulePattern: getModulePattern(target)) {
withBeatsEnv(archive: true, withModule: withModule, directory: directory) {
whenTrue(params.debug) {
dumpFilteredEnvironment()
dumpMage()
}
sh(label: "Make ${target}", script: "make ${target}")
sh(label: "Make ${target}", script: "make ${directoryFlag} ${target}")
whenTrue(clean) {
fixPermissions("${HOME}")
}
Expand All @@ -829,7 +831,7 @@ def mageTarget(Map args = [:]) {
def target = args.target
def withModule = args.get('withModule', false)
withGithubNotify(context: "${context}") {
withBeatsEnv(archive: true, withModule: withModule, modulePattern: getModulePattern(directory)) {
withBeatsEnv(archive: true, withModule: withModule, directory: directory) {
whenTrue(params.debug) {
dumpFilteredEnvironment()
dumpMage()
Expand All @@ -849,7 +851,7 @@ def mageTargetWin(Map args = [:]) {
def target = args.target
def withModule = args.get('withModule', false)
withGithubNotify(context: "${context}") {
withBeatsEnvWin(withModule: withModule, modulePattern: getModulePattern(directory)) {
withBeatsEnvWin(withModule: withModule, directory: directory) {
whenTrue(params.debug) {
dumpFilteredEnvironment()
dumpMageWin()
Expand All @@ -871,9 +873,10 @@ def getModulePattern(String toCompare) {
def withBeatsEnv(Map args = [:], Closure body) {
def archive = args.get('archive', true)
def withModule = args.get('withModule', false)
def directory = args.get('directory', '')
def modulePattern
if (withModule) {
modulePattern = args.containsKey('modulePattern') ? args.modulePattern : error('withBeatsEnv: modulePattern parameter is required.')
modulePattern = getModulePattern(directory)
}
def os = goos()
def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64"
Expand All @@ -882,7 +885,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")

// NOTE: This is required to run after the unstash
def module = withModule ? getCommonModuleInTheChangeSet(modulePattern) : ''
def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : ''

withEnv([
"HOME=${env.WORKSPACE}",
Expand Down Expand Up @@ -924,9 +927,10 @@ def withBeatsEnv(Map args = [:], Closure body) {

def withBeatsEnvWin(Map args = [:], Closure body) {
def withModule = args.get('withModule', false)
def directory = args.get('directory', '')
def modulePattern
if (withModule) {
modulePattern = args.containsKey('modulePattern') ? args.modulePattern : error('withBeatsEnvWin: modulePattern parameter is required.')
modulePattern = getModulePattern(directory)
}
final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin'
final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts'
Expand All @@ -936,7 +940,7 @@ def withBeatsEnvWin(Map args = [:], Closure body) {
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")

// NOTE: This is required to run after the unstash
def module = withModule ? getCommonModuleInTheChangeSet(modulePattern) : ''
def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : ''

withEnv([
"HOME=${env.WORKSPACE}",
Expand Down Expand Up @@ -1302,10 +1306,14 @@ def loadConfigEnvVars(){
For such, it's required to look for changes under the module folder and exclude anything else
such as ascidoc and png files.
*/
def getCommonModuleInTheChangeSet(String pattern) {
def getCommonModuleInTheChangeSet(String pattern, String directory) {
def module = ''
// Transform folder structure in regex format since path separator is required to be escaped
def transformedDirectory = directory.replaceAll('/', '\\/')
def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$"
def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)"
dir("${env.BASE_DIR}") {
module = getGitMatchingGroup(pattern: pattern , exclude: '^(((?!\\/module\\/).)*$|.*\\.asciidoc|.*\\.png)')
module = getGitMatchingGroup(pattern: pattern, exclude: exclude)
}
return module
}
Expand Down