Jenkinsfile.cloud

def DOCKER_IMG = "registry.fedoraproject.org/fedora:28"
def DOCKER_ARGS = "--net=host -v /srv:/srv --privileged"

// this var conveniently refers to a location on the server as well as the local dir we sync to/from
def server_dir = "${env.ARTIFACT_SERVER_DIR}"
def output = "${server_dir}/images"

def ref = "openshift/3.10/x86_64/os";

node(env.NODE) {
    checkout scm

    stage("Provision") {
        sh """
           rpm -q imagefactory-plugins-TinMan \
                  libguestfs-tools-c \
                  rpm-ostree \
                  rsync || \
               dnf -y install ostree \
                              imagefactory-plugins-TinMan \
                              libguestfs-tools-c \
                              rpm-ostree \
                              rsync
           """
    }

    stage("Sync In") {
        withCredentials([sshUserPrivateKey(credentialsId: env['ARTIFACT_SSH_CREDS_ID'],
                                           keyFileVariable: 'KEY_FILE')]) {
            sh """
                mkdir -p ${output}/cloud
                rsync -Hrlpt --stats \
                    -e 'ssh -i ${env.KEY_FILE} \
                            -o UserKnownHostsFile=/dev/null \
                            -o StrictHostKeyChecking=no' \
                    ${env.ARTIFACT_SERVER}:${server_dir}/images ${server_dir}
            """
        }

        // Also initialize an ostree repo for change detection
        sh """
            rm -rf repo && ostree init --repo=repo --mode=bare-user
            ostree --repo=repo remote add tmp --no-gpg-verify ${env.OSTREE_INSTALL_URL}
            ostree --repo=repo pull --commit-metadata-only tmp ${ref}
        """
    }

    // Check if there's a new version out.
    // Really, we should be checksumming e.g. the ks and tdl too.
    def latest_version, last_build_version, force_nocache
    stage("Check for Changes") {
        latest_version = sh(returnStdout: true,
                            script: "ostree show --repo=repo --print-metadata-key=version tmp:${ref}").trim()
        latest_version = latest_version[1..-2] // trim single quotes
        if (fileExists('force-nocache-build')) {
            force_nocache = readFile('force-nocache-build').trim();
        }
        last_build_version = sh(returnStdout: true,
                                script: "cat ${output}/cloud/version.txt || :").trim()
    }

    if (latest_version == last_build_version && latest_version != force_nocache) {
        currentBuild.result = 'SUCCESS'
        currentBuild.description = '(No changes)'
        return
    }

    stage("Prepare Configs") {
        sh "sed -i 's,\\(<url>\\).*\\(<\\/url\\),\\1${env.INSTALLER_TREE_URL}\\2,' rhcos.tdl"
        sh "sed -i 's,@@OSTREE_INSTALL_URL@@,${env.OSTREE_INSTALL_URL},' cloud.ks"
        sh "sed -i 's,@@OSTREE_INSTALL_REF@@,${ref},' cloud.ks"
        sh "rm -rf /var/lib/imagefactory/storage/*"
    }

    stage("Running imagefactory") {
        sh """
            imagefactory --debug base_image \
                --file-parameter install_script cloud.ks \
                --parameter offline_icicle True \
                rhcos.tdl
        """
    }

    def commit, version, dirpath, qcow, vmdk
    stage("Finalizing") {
        def image = sh(returnStdout: true, script: "ls /var/lib/imagefactory/storage/*.body").trim()
        // just introspect after the fact to avoid race conditions
        commit = sh(returnStdout: true,
                    script: """LIBGUESTFS_BACKEND=direct virt-cat -a ${image} -m /dev/coreos/root:/ \
                                    /ostree/repo/refs/remotes/rhcos/${ref}""").trim()

        // do this again *after* running imgfac to close race
        // fetch parent too for the later db diff
        sh "ostree --repo=repo pull --depth=1 --commit-metadata-only tmp ${commit}"
        version = sh(returnStdout: true,
                     script: "ostree show --repo=repo --print-metadata-key=version ${commit}").trim()
        assert (version.startsWith("'") && version.endsWith("'"))
        version = version[1..-2] // trim single quotes

        currentBuild.description = "${version} (${commit})"

        def dirname = (new Date()).format("YYYY-MM-dd-HH-mm-ss-") + commit
        dirpath = "${output}/cloud/${dirname}"
        qcow = "${dirpath}/rhcos.qcow2"
        vmdk = "${dirpath}/rhcos.vmdk"
        sh "mkdir -p ${dirpath}"
        // this belongs better in a JSON file, but for now just use a file;
        // this is used higher up to determine no-op changes
        sh "echo '${version}' > ${dirpath}/version.txt"

        sh "qemu-img convert -f raw -O qcow2 ${image} ${qcow}"
        sh "gzip ${qcow}"
        sh "qemu-img convert -f raw -O vmdk ${image} -o adapter_type=lsilogic,subformat=streamOptimized,compat6 ${vmdk}"

        sh "ln -sfn ${dirname} ${output}/cloud/latest"
        // just keep the last 2 (+ latest symlink)
        sh "cd ${output}/cloud && (ls | head -n -3 | xargs -r rm -rf)"
    }

    stage("Generate Metadata") {
        // we're stuck with an older host because imagefactory produces kernel
        // panics on newer hosts, so let's use a container to have access to a
        // newer rpm-ostree that understands the new pkglist metadata
        docker.image(DOCKER_IMG).inside(DOCKER_ARGS) {
            sh "dnf install -y rpm-ostree ostree"

            def prev_commit = sh(returnStdout: true,
                                 script: "ostree --repo=repo rev-parse ${commit}^").trim()

            sh """
                rpm-ostree db list --repo=repo ${commit} > \
                    /${dirpath}/pkglist.txt
            """
            sh """
                rpm-ostree db diff --repo=repo \
                    ${prev_commit} ${commit} > ${dirpath}/pkgdiff.txt
            """
            sh "sha256sum ${qcow}.gz | awk '{print \$1}' > ${qcow}.gz.sha256sum"
            sh "sha256sum ${vmdk} | awk '{print \$1}' > ${vmdk}.sha256sum"
        }
    }

    stage("Sync Out") {
        withCredentials([sshUserPrivateKey(credentialsId: env['ARTIFACT_SSH_CREDS_ID'],
                                           keyFileVariable: 'KEY_FILE')]) {
            sh """
                rsync -Hrlpt --stats --delete --delete-after \
                    -e 'ssh -i ${env.KEY_FILE} \
                            -o UserKnownHostsFile=/dev/null \
                            -o StrictHostKeyChecking=no' \
                    ${output}/ ${env.ARTIFACT_SERVER}:${output}
            """
        }
    }

    stage("Create AMI") {
        docker.image(DOCKER_IMG).inside(DOCKER_ARGS) {
            // not packaged yet, just build from src for now
            sh "dnf install -y git golang awscli jq"
            sh "git clone https://github.com/coreos/mantle"
            sh "mantle/build ore"

            withCredentials([[$class: 'AmazonWebServicesCredentialsBinding',
                              credentialsId: env['AWS_CREDENTIALS']]]) {
                sh """
                    # use a symlink so that our uploaded filename is unique
                    ln -sf ${output}/cloud/latest/rhcos.vmdk rhcos-${commit}.vmdk
                    mantle/bin/ore aws upload --region us-east-1 \
                        --ami-name 'rhcos_dev_${commit[0..6]}' \
                        --ami-description 'Red Hat CoreOS ${version} (${commit})' \
                        --bucket 's3://${env.S3_PRIVATE_BUCKET}/rhcos/cloud' \
                        --file rhcos-${commit}.vmdk \
                        --name "rhcos_dev_${commit[0..6]}" \
                        --delete-object | tee out.json # already default, just being explicit

                    # Add the version and commit as tags to both the AMI and the underlying snapshot.
                    # We should teach mantle to apply extra tags.
                    ami=\$(jq -r .HVM out.json)
                    snapshot=\$(jq -r .SnapshotID out.json)
                    AWS_DEFAULT_REGION=us-east-1 aws ec2 create-tags \
                        --resources \${ami} \${snapshot} \
                        --tags Key=ostree_commit,Value=${commit} \
                               Key=ostree_version,Value=${version}
                """
            }
        }
    }
}