Introducing some Job DSL support

Vagrantfile

@@ -4,6 +4,11 @@ Vagrant.configure("2") do |config|
   # Popular base box, should work!
   config.vm.box = "ubuntu/trusty64"
 
+  config.vm.provider "virtualbox" do |v|
+    v.memory = 2048
+    v.cpus = 2
+  end
+
   # Forward expected guest Jenkins to host 8080
   config.vm.network "forwarded_port", guest: 8080, host: 8080
 

build.gradle

@@ -12,6 +12,7 @@ idea {
 }
 
 repositories {
+    jcenter()
     maven { url 'https://repo.jenkins-ci.org/releases/'}
     maven { url 'https://repo.jenkins-ci.org/public/'}
     mavenCentral()
@@ -21,21 +22,35 @@ sourceSets {
     // Additional source set for the init scripts embedded within the plugins image (TODO: Move to its own image?)
     initScripts {
         groovy {
-            srcDir 'images/jenkins-plugins/files/init.groovy.d'
+            srcDirs 'images/jenkins-plugins/files/init.groovy.d'
+            compileClasspath += main.compileClasspath
+        }
+    }
+
+    jobs {
+        groovy {
+            srcDirs 'jobs'
+            compileClasspath += main.compileClasspath
         }
     }
 }
 
 dependencies {
     // The basics - Groovy, Jenkins, etc
     compile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.4.8'
-    compile group: 'org.jenkins-ci.main', name: 'jenkins-core', version: '2.121.1'
+    compile group: 'org.jenkins-ci.main', name: 'jenkins-core', version: '2.138.2'
     compile group: 'javax.servlet', name: 'javax.servlet-api', version: '3.1.0'
     testCompile group: 'junit', name: 'junit', version: '4.12'
 
     // General Jenkins plugins - TODO: Parse out of plugins.txt transitively somehow and translate to Maven style deps
     compile group: 'org.jenkins-ci.plugins', name: 'credentials', version: '2.1.13', ext: 'jar'
 
+    // Job DSL and plugins used in examples
+    compile group: 'org.jenkins-ci.plugins', name: 'job-dsl', version: '1.70', ext: 'jar'
+    compile group: 'org.jenkins-ci.plugins', name: 'cloudbees-folder', version: '6.7', ext: 'jar'
+    compile group: 'org.jenkins-ci.plugins', name: 'gradle', version: '1.29', ext: 'jar'
+    compile group: 'org.jenkins-ci.plugins', name: 'email-ext', version: '2.63', ext: 'jar'
+
     // Optional: Place non-hosted plugins in the lib dir to have them recognized within the IDE
     compile fileTree(dir: 'lib', include: ['*.jar'])
 }

deploy/master/docker-compose.yml

@@ -16,6 +16,7 @@ services:
       - plugins:/usr/share/jenkins/ref/plugins
       - warfile:/usr/share/jenkins/ref/warfile
       - groovy:/var/jenkins_home/init.groovy.d
+      - dsl:/var/jenkins_home/dslScripts
       - ${PWD}/../../secure:/secure:ro
 
   # Jenkins plugins' configuration
@@ -25,10 +26,12 @@ services:
       - plugins:/usr/share/jenkins/ref/plugins
       - warfile:/usr/share/jenkins/ref/warfile
       - groovy:/usr/share/jenkins/ref/init.groovy.d
+      - dsl:/usr/share/jenkins/ref/dslScripts
 
 # Define named volumes. These are what we use to share the data from one
 # container to another, thereby making our jenkins.war and plugins available
 volumes:
   plugins:
   warfile:
   groovy:
+  dsl:

images/jenkins-base/Dockerfile

@@ -17,13 +17,16 @@ ENV uid=1000
 ENV gid=1000
 
 # Jenkins Version info
-ENV JENKINS_VERSION 2.121.1
-ENV JENKINS_SHA 5bb075b81a3929ceada4e960049e37df5f15a1e3cfc9dc24d749858e70b48919
+ENV JENKINS_VERSION 2.138.2
+ENV JENKINS_SHA d8ed5a7033be57aa9a84a5342b355ef9f2ba6cdb490db042a6d03efb23ca1e83
 
 # These URLs can be swapped out for internal repos if needed. Secrets required may vary :)
 ENV JENKINS_UC https://updates.jenkins.io
 ENV JENKINS_URL https://repo.jenkins-ci.org/public/org/jenkins-ci/main/jenkins-war/${JENKINS_VERSION}/jenkins-war-${JENKINS_VERSION}.war
 
+# Optional extra hook for Job DSL examples, used in one of the init scripts if present. No env var == no job DSL set up
+ENV DSL_REPO https://github.com/sheehan/job-dsl-gradle-example.git
+
 # Jenkins is run with user `jenkins`, uid = 1000
 # If you bind mount a volume from the host or a data container,
 # ensure you use the same uid

images/jenkins-master/Dockerfile

@@ -14,6 +14,9 @@ ENV COPY_REFERENCE_FILE_LOG $JENKINS_HOME/copy_reference_file.log
 # or config file with your custom jenkins Docker image.
 RUN mkdir -p /usr/share/jenkins/ref/init.groovy.d
 
+# Do the thing but for DSL scripts
+RUN mkdir -p /usr/share/jenkins/ref/dslScripts
+
 # # Disable the upgrade banner & admin pw (we will add one later)
 RUN echo 2.0 > /usr/share/jenkins/ref/jenkins.install.UpgradeWizard.state \
     && echo 2.0 > ${JENKINS_HOME}/jenkins.install.InstallUtil.lastExecVersion

images/jenkins-plugins/Dockerfile

@@ -14,6 +14,9 @@ ADD files/install-plugins.sh /usr/local/bin/
 # Add our groovy init files
 ADD files/init.groovy.d /usr/share/jenkins/ref/init.groovy.d
 
+# Add our Job DSL files
+ADD files/dslScripts /usr/share/jenkins/ref/dslScripts
+
 # Download the Jenkins war
 # JENKINS_URL, JENKINS_ROOT, JENKINS_WAR, and JENKINS_SHA are set in the parent
 RUN mkdir -p ${JENKINS_ROOT}/ref/warfile \
@@ -24,7 +27,7 @@ RUN mkdir -p ${JENKINS_ROOT}/ref/warfile \
 # We will run all of this as the jenkins user as is dictated by the base imge
 USER ${user}
 
-# Install our base set of plugins and their depdendencies that are listed in
+# Install our base set of plugins and their dependencies that are listed in
 # plugins.txt
 ADD files/plugins.txt /tmp/plugins-main.txt
 RUN install-plugins.sh `cat /tmp/plugins-main.txt`
@@ -33,6 +36,7 @@ RUN install-plugins.sh `cat /tmp/plugins-main.txt`
 VOLUME /usr/share/jenkins/ref/plugins
 VOLUME /usr/share/jenkins/ref/warfile
 VOLUME /usr/share/jenkins/ref/init.groovy.d
+VOLUME /usr/share/jenkins/ref/dslScripts
 
 # It's easy to get confused when just a volume is being used, so let's just keep
 # the container alive for clarity. This entrypoint will keep the container

images/jenkins-plugins/files/dslScripts/GentleRestartJenkins.groovy

@@ -0,0 +1,77 @@
+job('JenkinsGentleRestart') {
+    description('Gently swaddles Jenkins into goodnight mode, sings it a lullaby while waiting for any jobs to finish, then restarts.')
+    label("master")
+    steps {
+        systemGroovyCommand ('''
+            /*
+                Copyright (c) 2015-2018 Sam Gleske - https://github.com/samrocketman/jenkins-script-console-scripts
+
+                Permission is hereby granted, free of charge, to any person obtaining a copy of
+                this software and associated documentation files (the "Software"), to deal in
+                the Software without restriction, including without limitation the rights to
+                use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+                the Software, and to permit persons to whom the Software is furnished to do so,
+                subject to the following conditions:
+
+                The above copyright notice and this permission notice shall be included in all
+                copies or substantial portions of the Software.
+
+                THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+                IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+                FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+                COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+                IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+                CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+            */
+            /*
+               This script starts a background thread which will wait for Jenkins to finish
+               executing jobs before restarting.  The thread will abort if shutdown mode is
+               disabled before jobs finish.
+
+               Tested on Jenkins ver. 2.7.1
+            */
+
+            import hudson.model.RestartListener
+            import java.util.logging.Level
+            import java.util.logging.Logger
+            import jenkins.model.*
+
+            //user configurable variable
+            if(!binding.hasVariable('timeout_seconds')) {
+                timeout_seconds = 10
+            }
+
+            if(timeout_seconds in String) {
+                timeout_seconds = Integer.decode(timeout_seconds)
+            }
+
+            //type check user defined parameters/bindings
+            if(!(timeout_seconds in Integer)) {
+                throw new Exception('PARAMETER ERROR: timeout_seconds must be an integer.')
+            }
+
+            Logger logger = Logger.getLogger('jenkins.instance.restart')
+            Jenkins.instance.doQuietDown();
+
+            //start a background thread
+            def thread = Thread.start {
+                logger.log(Level.INFO, "Jenkins safe restart initiated.")
+                while(true) {
+                    if(Jenkins.instance.isQuietingDown()) {
+                        if(RestartListener.isAllReady()) {
+                            Jenkins.instance.restart()
+                        }
+                        logger.log(Level.INFO, "Jenkins jobs are not idle.  Waiting ${timeout_seconds} seconds before next restart attempt.")
+                        sleep(timeout_seconds*1000)
+                    }
+                    else {
+                        logger.log(Level.INFO, "Shutdown mode not enabled.  Jenkins restart aborted.")
+                        break
+                    }
+                }
+            }
+
+            println 'A safe restart has been scheduled.  See the Jenkins logs for restart status updates.  Logger is jenkins.instance.restart.'
+        ''')
+    }
+}

images/jenkins-plugins/files/init.groovy.d/03_PrepDSL.groovy

@@ -0,0 +1,30 @@
+import hudson.model.Cause.UserIdCause
+import hudson.model.FreeStyleProject
+import javaposse.jobdsl.dsl.DslScriptLoader
+import javaposse.jobdsl.plugin.JenkinsJobManagement
+import jenkins.model.Jenkins
+
+// Make the Job DSL integration optional. Fairly easy to come up with different ways to trigger it. Env vars are simple.
+if (! System.getenv("DSL_REPO")) {
+    println "No DSL repo defined, skipping DSL prep"
+    return
+}
+
+// Create the seed job itself, so it can run and create the "real" DSL-managed jobs
+def jobDslScript = new File("/var/jenkins_home/init.groovy.d/SeedJobDSL.groovy.DSL")
+def workspace = new File("/tmp/dsl")
+def jobManagement = new JenkinsJobManagement(System.out, [:], workspace)
+new DslScriptLoader(jobManagement).runScript(jobDslScript.text)
+
+// Schedule the job to run so the jobs are created when Jenkins starts - or skip this for more control
+Jenkins.instance.getItemByFullName('seed-job', FreeStyleProject.class).scheduleBuild(new UserIdCause())
+
+
+// Secondary job seeder using a local dir
+def jobDslScript2 = new File("/var/jenkins_home/init.groovy.d/SeedJobDSL2.groovy.DSL")
+def workspace2 = new File("/tmp/dsl")
+def jobManagement2 = new JenkinsJobManagement(System.out, [:], workspace2)
+new DslScriptLoader(jobManagement2).runScript(jobDslScript2.text)
+
+// Schedule the job to run so the jobs are created when Jenkins starts.
+Jenkins.instance.getItemByFullName('seed-job2', FreeStyleProject.class).scheduleBuild(new UserIdCause())

images/jenkins-plugins/files/init.groovy.d/SeedJobDSL.groovy.DSL

@@ -0,0 +1,22 @@
+job('seed-job') {
+    description('Seed Job to create other DSL-based jobs')
+    label("master")
+    scm {
+        git("${System.getenv("DSL_REPO")}")
+    }
+    triggers {
+        // Every 15 minutes poll to see if there are new changes
+        scm('H/15 * * * *')
+        // Once a day run anyway to catch and highlight manual changes
+        cron('H/60 H/24 * * *')
+    }
+    concurrentBuild(false)
+    steps {
+        dsl {
+            external "src/jobs/**/*.groovy"
+            removeAction('DELETE')
+            removeViewAction('DELETE')
+            ignoreExisting(false)
+        }
+    }
+}

images/jenkins-plugins/files/init.groovy.d/SeedJobDSL2.groovy.DSL

@@ -0,0 +1,14 @@
+job('seed-job2') {
+    description('Seed Job to create other DSL-based jobs')
+    label("master")
+    customWorkspace('/var/jenkins_home/dslScripts')
+    concurrentBuild(false)
+    steps {
+        dsl {
+            external "GentleRestartJenkins.groovy"
+            removeAction('DELETE')
+            removeViewAction('DELETE')
+            ignoreExisting(false)
+        }
+    }
+}

images/jenkins-plugins/files/plugins.txt

@@ -2,3 +2,8 @@ credentials
 github
 greenballs
 groovy
+workflow-support
+job-dsl
+cloudbees-folder
+gradle
+email-ext

src/jobs/examplejob.groovy

@@ -0,0 +1,16 @@
+def project = 'quidryan/aws-sdk-test'
+def branchApi = new URL("https://api.github.com/repos/${project}/branches")
+def branches = new groovy.json.JsonSlurper().parse(branchApi.newReader())
+
+branches.each {
+    def branchName = it.name
+    def jobName = "${project}-${branchName}".replaceAll('/','-')
+    job(jobName) {
+        scm {
+            git("git://github.com/${project}.git", branchName)
+        }
+        steps {
+            maven("test -Dproject.name=${project}/${branchName}")
+        }
+    }
+}

src/main/resources/idea.gdsl

@@ -0,0 +1,8 @@
+// enable DSL support in IDEA, see https://confluence.jetbrains.com/display/GRVY/Scripting+IDE+for+DSL+awareness
+
+def jobPath = /.*\/jobs\/.*\.groovy/
+
+def ctx = context(pathRegexp: jobPath)
+contributor(ctx, {
+    delegatesTo(findClass('javaposse.jobdsl.dsl.DslFactory'))
+})