From e91a1c89c404b46f7332b061c152d110a6f47cc7 Mon Sep 17 00:00:00 2001 From: Brian Peiris Date: Thu, 7 Jan 2021 17:42:53 -0500 Subject: [PATCH 1/2] Jenkins changes for promotion to staging channel --- Jenkinsfile | 105 ++++++++++++++++++++++++---------- scripts/hab-build-and-push.sh | 8 ++- 2 files changed, 81 insertions(+), 32 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b3853332f..a8b1d50ab 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,19 +1,5 @@ import groovy.json.JsonOutput -// From https://issues.jenkins-ci.org/browse/JENKINS-44231 - -// Given arbitrary string returns a strongly escaped shell string literal. -// I.e. it will be in single quotes which turns off interpolation of $(...), etc. -// E.g.: 1'2\3\'4 5"6 (groovy string) -> '1'\''2\3\'\''4 5"6' (groovy string which can be safely pasted into shell command). -def shellString(s) { - // Replace ' with '\'' (https://unix.stackexchange.com/a/187654/260156). Then enclose with '...'. - // 1) Why not replace \ with \\? Because '...' does not treat backslashes in a special way. - // 2) And why not use ANSI-C quoting? I.e. we could replace ' with \' - // and enclose using $'...' (https://stackoverflow.com/a/8254156/4839573). - // Because ANSI-C quoting is not yet supported by Dash (default shell in Ubuntu & Debian) (https://unix.stackexchange.com/a/371873). - '\'' + s.replace('\'', '\'\\\'\'') + '\'' -} - pipeline { agent any @@ -43,14 +29,36 @@ pipeline { def slackURL = env.SLACK_URL def sentryDsn = env.SENTRY_DSN def gaTrackingId = env.GA_TRACKING_ID + def buildNumber = env.BUILD_NUMBER + def gitCommit = env.GIT_COMMIT + def jobName = env.JOB_NAME + def disableDeploy = env.DISABLE_DEPLOY + def promoteToChannel = env.PROMOTE_TO_CHANNEL - def habCommand = "/bin/bash scripts/hab-build-and-push.sh \\\"${baseAssetsPath}\\\" \\\"${hubsServer}\\\" \\\"${reticulumServer}\\\" \\\"${thumbnailServer}\\\" \\\"${corsProxyServer}\\\" \\\"${nonCorsProxyDomains}\\\" \\\"${sentryDsn}\\\" \\\"${gaTrackingId}\\\" \\\"${targetS3Bucket}\\\" \\\"${isMoz}\\\" \\\"${env.BUILD_NUMBER}\\\" \\\"${env.GIT_COMMIT}\\\"" - sh "/usr/bin/script --return -c ${shellString(habCommand)} /dev/null" + def habCommand = ( + "/bin/bash scripts/hab-build-and-push.sh " + + "\\\"${baseAssetsPath}\\\" " + + "\\\"${hubsServer}\\\" " + + "\\\"${reticulumServer}\\\" " + + "\\\"${thumbnailServer}\\\" " + + "\\\"${corsProxyServer}\\\" " + + "\\\"${nonCorsProxyDomains}\\\" " + + "\\\"${sentryDsn}\\\" " + + "\\\"${gaTrackingId}\\\" " + + "\\\"${targetS3Bucket}\\\" " + + "\\\"${isMoz}\\\" " + + "\\\"${buildNumber}\\\" " + + "\\\"${gitCommit}\\\" " + + "\\\"${disableDeploy}\\\" " + ) + runCommand(habCommand) - def s = $/eval 'ls -rt results/*.hart | head -n 1'/$ + def s = $/eval 'ls -t results/*.hart | head -n 1'/$ def hart = sh(returnStdout: true, script: "${s}").trim() + s = $/eval 'tail -n +6 ${hart} | xzcat | tar tf - | grep IDENT'/$ def identPath = sh(returnStdout: true, script: "${s}").trim() + s = $/eval 'tail -n +6 ${hart} | xzcat | tar xf - "${identPath}" -O'/$ def packageIdent = sh(returnStdout: true, script: "${s}").trim() def packageTimeVersion = packageIdent.tokenize('/')[3] @@ -59,22 +67,57 @@ pipeline { def gitMessage = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'[%an] %s'").trim() def gitSha = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim() - def text = ( - "** *${env.JOB_NAME}* " + - " ${spokeVersion} " + - "Spoke: ```${gitSha} ${gitMessage}```\n" + - "<${smokeURL}?required_version=${spokeVersion}|Smoke Test> - to push:\n" + - "`/mr spoke deploy ${spokeVersion} s3://${targetS3Bucket}`" - ) - def payload = 'payload=' + JsonOutput.toJson([ - text : text, - channel : "#mr-builds", - username : "buildbot", - icon_emoji: ":gift:" - ]) - sh "curl -X POST --data-urlencode ${shellString(payload)} ${slackURL}" + + if (promoteToChannel != null && promoteToChannel != "") { + runCommand("sudo /usr/bin/hab-ret-pkg-promote ${packageIdent} ${promoteToChannel}") + + def text = ( + "** *${jobName}* " + + "\n" + + " " + + "Promoted ${spokeVersion} to ${promoteToChannel}: ```${gitSha} ${gitMessage}```\n" + ) + sendSlackMessage(text, "#mr-builds", ":gift:", slackURL) + } else { + def text = ( + "** *${jobName}* " + + " ${spokeVersion} " + + "Spoke: ```${gitSha} ${gitMessage}```\n" + + "<${smokeURL}?required_version=${spokeVersion}|Smoke Test> - to push:\n" + + "`/mr spoke deploy ${spokeVersion} s3://${targetS3Bucket}`" + ) + sendSlackMessage(text, "#mr-builds", ":gift:", slackURL) + } } } } } } + +def runCommand(command) { + sh "/usr/bin/script --return -c ${shellString(command)} /dev/null" +} + +def sendSlackMessage(text, channel, icon, slackURL) { + def payload = 'payload=' + JsonOutput.toJson([ + text : text, + channel : channel, + username : "buildbot", + icon_emoji: icon + ]) + sh "curl -X POST --data-urlencode ${shellString(payload)} ${slackURL}" +} + +// From https://issues.jenkins-ci.org/browse/JENKINS-44231 + +// Given arbitrary string returns a strongly escaped shell string literal. +// I.e. it will be in single quotes which turns off interpolation of $(...), etc. +// E.g.: 1'2\3\'4 5"6 (groovy string) -> '1'\''2\3\'\''4 5"6' (groovy string which can be safely pasted into shell command). +def shellString(s) { + // Replace ' with '\'' (https://unix.stackexchange.com/a/187654/260156). Then enclose with '...'. + // 1) Why not replace \ with \\? Because '...' does not treat backslashes in a special way. + // 2) And why not use ANSI-C quoting? I.e. we could replace ' with \' + // and enclose using $'...' (https://stackoverflow.com/a/8254156/4839573). + // Because ANSI-C quoting is not yet supported by Dash (default shell in Ubuntu & Debian) (https://unix.stackexchange.com/a/371873). + '\'' + s.replace('\'', '\'\\\'\'') + '\'' +} diff --git a/scripts/hab-build-and-push.sh b/scripts/hab-build-and-push.sh index 13e6c4dfa..fe139f652 100755 --- a/scripts/hab-build-and-push.sh +++ b/scripts/hab-build-and-push.sh @@ -12,6 +12,7 @@ export TARGET_S3_BUCKET=$9 export IS_MOZ=${10} export BUILD_NUMBER=${11} export GIT_COMMIT=${12} +export DISABLE_DEPLOY=${13} export BUILD_VERSION="${BUILD_NUMBER} (${GIT_COMMIT})" export SENTRY_LOG_LEVEL=debug @@ -38,6 +39,11 @@ sudo /usr/bin/hab-pkg-install results/*.hart hab svc load $PKG hab svc stop $PKG +DEPLOY_TYPE="s3" +if [[ DISABLE_DEPLOY == "true" ]]; then + DEPLOY_TYPE="none" +fi + # Apparently these vars come in from jenkins with quotes already cat > build-config.toml << EOTOML [general] @@ -52,7 +58,7 @@ ga_tracking_id = $GA_TRACKING_ID is_moz = $IS_MOZ [deploy] -type = "s3" +type = "$DEPLOY_TYPE" target = $TARGET_S3_BUCKET region = "us-west-1" EOTOML From 15d4a07742240f157a690188516ff59836c4fb38 Mon Sep 17 00:00:00 2001 From: Brian Peiris Date: Mon, 11 Jan 2021 19:34:23 -0500 Subject: [PATCH 2/2] have jenkins script build and message qa packages --- Jenkinsfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a8b1d50ab..011a37b06 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,7 +33,8 @@ pipeline { def gitCommit = env.GIT_COMMIT def jobName = env.JOB_NAME def disableDeploy = env.DISABLE_DEPLOY - def promoteToChannel = env.PROMOTE_TO_CHANNEL + def showQAPromoteCommand = env.SHOW_QA_PROMOTE_COMMAND + def qaBuildsSlackChannel = env.QA_BUILDS_SLACK_CHANNEL def habCommand = ( "/bin/bash scripts/hab-build-and-push.sh " @@ -68,16 +69,15 @@ pipeline { def gitMessage = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'[%an] %s'").trim() def gitSha = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim() - if (promoteToChannel != null && promoteToChannel != "") { - runCommand("sudo /usr/bin/hab-ret-pkg-promote ${packageIdent} ${promoteToChannel}") - + if (showQAPromoteCommand == "true") { def text = ( "** *${jobName}* " + - "\n" + - " " + - "Promoted ${spokeVersion} to ${promoteToChannel}: ```${gitSha} ${gitMessage}```\n" + " ${spokeVersion} " + + "Spoke: ```${gitSha} ${gitMessage}```\n" + + "${packageIdent} built and uploaded - to promote:\n" + + "`/mr promote-spoke-qa ${packageIdent}`" ) - sendSlackMessage(text, "#mr-builds", ":gift:", slackURL) + sendSlackMessage(text, qaBuildsSlackChannel, ":gift:", slackURL) } else { def text = ( "** *${jobName}* " +