diff --git a/jobs/build-kola-containers.Jenkinsfile b/jobs/build-kola-containers.Jenkinsfile new file mode 100644 index 000000000..8335585d8 --- /dev/null +++ b/jobs/build-kola-containers.Jenkinsfile @@ -0,0 +1,223 @@ +def gitref, commit, shortcommit +node { + checkout scm + // these are script global vars + pipeutils = load("utils.groovy") +} + +properties([ + pipelineTriggers([ + [$class: 'GenericTrigger', + genericVariables: [ + [ + key: 'COREOS_ASSEMBLER_GIT_REF', + value: '$.ref', + expressionType: 'JSONPath', + regexpFilter: 'refs/heads/', //Optional, defaults to empty string + defaultValue: '' //Optional, defaults to empty string + ] + ], + causeString: 'Triggered on $ref', + token: 'build-kola-test-containers', + tokenCredentialId: '', + printContributedVariables: true, + printPostContent: true, + silentResponse: false, + regexpFilterText: '$COREOS_ASSEMBLER_GIT_REF', + regexpFilterExpression: 'main|rhcos-.*' + ] + ]), + parameters([ + string(name: 'ARCHES', + description: 'Space-separated list of target architectures', + defaultValue: "x86_64" + " " + pipeutils.get_supported_additional_arches().join(" "), + trim: true), + string(name: 'COREOS_ASSEMBLER_GIT_URL', + description: 'Override the coreos-assembler git repo to use', + //defaultValue: "https://github.com/coreos/coreos-assembler.git", + defaultValue: "https://github.com/jbtrystram/coreos-assembler.git", + trim: true), + string(name: 'COREOS_ASSEMBLER_GIT_REF', + description: 'Override the coreos-assembler git ref to use', + //defaultValue: "main", + defaultValue: "coreos-owned-test-images", + trim: true), + string(name: 'CONTAINER_REGISTRY_ORG', + description: 'Override the registry org to push the containers to', + //defaultValue: "quay.io/coreos-assembler", + defaultValue: "quay.io/jbtrystramtestimages", + trim: true), + string(name: 'PATH_TO_CONTEXTS', + description: """Override the path to the contexts directories to use as build contexts. + Each directory should contain a containerfile and the image will be named after the directory name.""", + defaultValue: "tests/containers/", + trim: true) + ]), + buildDiscarder(logRotator( + numToKeepStr: '100', + artifactNumToKeepStr: '100' + )), + durabilityHint('PERFORMANCE_OPTIMIZED') +]) + +node { + change = checkout( + changelog: true, + poll: false, + scm: [ + $class: 'GitSCM', + branches: [[name: "origin/${params.COREOS_ASSEMBLER_GIT_REF}"]], + userRemoteConfigs: [[url: params.COREOS_ASSEMBLER_GIT_URL]], + extensions: [[$class: 'CloneOption', + noTags: true, + reference: '', + shallow: true]] + ] + ) + + gitref = params.COREOS_ASSEMBLER_GIT_REF + def output = shwrapCapture("git rev-parse HEAD") + commit = output.substring(0,40) + shortcommit = commit.substring(0,7) +} + +node { + change = checkout( + changelog: true, + poll: false, + scm: [ + $class: 'GitSCM', + branches: [[name: "origin/${params.COREOS_ASSEMBLER_GIT_REF}"]], + userRemoteConfigs: [[url: params.COREOS_ASSEMBLER_GIT_URL]], + extensions: [[$class: 'CloneOption', + noTags: true, + reference: '', + shallow: true]] + ] + ) + + gitref = params.COREOS_ASSEMBLER_GIT_REF + def output = shwrapCapture("git rev-parse HEAD") + commit = output.substring(0,40) + shortcommit = commit.substring(0,7) +} + +node { + def path=params.PATH_TO_CONTEXTS + // gather the context folders list + contexts = sh(returnStdout: true, script: """ + cd ${path} + find . -maxdepth 1 -mindepth 1 -type d -not -path '*/\\.*' -exec basename {} \\; + """).trim().split("\n") +} + +currentBuild.description = "[${gitref}@${shortcommit}] Waiting" + +// Get the list of requested architectures to build for +def basearches = params.ARCHES.split() as Set +// and the list of contexts to build +def contexts = contexts as Set + +lock(resource: "build-containers") { + cosaPod(image: "quay.io/coreos-assembler/coreos-assembler", + memory: "512Mi", kvm: false, + serviceAccount: "jenkins") { + timeout(time: 60, unit: 'MINUTES') { + try { + + currentBuild.description = "[${gitref}@${shortcommit}] Running" + + // By default we will allow re-using cache layers for one day. + // This is mostly so we can prevent re-downloading the RPMS + // and repo metadata and over again in a given day for successive + // builds. + def cacheTTL = "24h" + def force = "" + if (params.FORCE) { + force = '--force' + // Also set cacheTTL to 0.1s to allow users an escape hatch + // to force no cache layer usage. + cacheTTL = "0.1s" + } + + withCredentials([file(credentialsId: 'jbtrystram-testorg-robot-push-secret', variable: 'REGISTRY_SECRET')]) { + stage('Build Containers') { + parallel basearches.collectEntries{arch -> [arch, { + for (directory in contexts) { + def dir = params.PATH_TO_CONTEXTS + directory + pipeutils.withPodmanRemoteArchBuilder(arch: arch) { + shwrap(""" + cosa remote-build-container \ + --git-sub-dir $dir\ + --arch $arch --cache-ttl ${cacheTTL} \ + --git-ref $commit ${force} \ + --git-url ${params.COREOS_ASSEMBLER_GIT_URL} \ + --repo ${params.CONTAINER_REGISTRY_ORG}/$directory \ + --push-to-registry --auth=\$REGISTRY_SECRET + """) + } + } + }]} + } + + stage('Push manifests') { + for (architecture in basearches) { + def arch = architecture + for (directory in contexts) { + def dir = directory + images += " --image=docker://${params.CONTAINER_REGISTRY_ORG}/${dir}:${arch}-${shortcommit}" + } + } + + for (directory in contexts) { + def dir = directory + shwrap(""" + export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507 + cosa push-container-manifest --v2s2 \ + --auth=\$REGISTRY_SECRET --tag ${gitref} \ + --repo ${params.CONTAINER_REGISTRY_ORG}/$dir ${images} + """) + // Specifically for the `main` branch let's also update the `latest` tag + // If there was a way to alias/tie these two together in the Quay UI + // that would be preferable. + if (gitref == "main") { + shwrap(""" + export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507 + skopeo copy --all --authfile \$REGISTRY_SECRET \ + docker://${params.CONTAINER_REGISTRY_ORG}/$dir:main \ + docker://${params.CONTAINER_REGISTRY_ORG}/$dir:latest + """) + } + } + } + + + stage('Delete Intermediate Tags') { + for (directory in contexts) { + def dir = directory + shwrap(""" + export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507 + skopeo delete --authfile=\$REGISTRY_SECRET \ + docker://${params.CONTAINER_REGISTRY_ORG}/$dir:${arch}-${shortcommit} + """) + } + } + + currentBuild.result = 'SUCCESS' + + } catch (e) { + currentBuild.result = 'FAILURE' + throw e + } finally { + if (currentBuild.result == 'SUCCESS') { + currentBuild.description = "[${gitref}@${shortcommit}] ⚡" + } else { + currentBuild.description = "[${gitref}@${shortcommit}] ❌" + } + if (currentBuild.result != 'SUCCESS') { + message = "build-kola-test-containers #${env.BUILD_NUMBER} <${env.BUILD_URL}|:jenkins:> <${env.RUN_DISPLAY_URL}|:ocean:> [${gitref}@${shortcommit}]" + pipeutils.trySlackSend(message: message) + } + } +}}} // cosaPod, timeout, and lock finish here +