Add a new Jenkins job to build containers for kola

Some kola tests require external images. This job will build
and publish the images present under the test/containers path
of coreos-assembler.

Inital PR for coreos-assembler: coreos/coreos-assembler#3727

x-ref coreos/fedora-coreos-tracker#1639

jobs/build-kola-containers.Jenkinsfile

@@ -0,0 +1,223 @@
+def gitref, commit, shortcommit
+node {
+    checkout scm
+    // these are script global vars
+    pipeutils = load("utils.groovy")
+}
+
+properties([
+    pipelineTriggers([
+        [$class: 'GenericTrigger',
+         genericVariables: [
+          [
+           key: 'COREOS_ASSEMBLER_GIT_REF',
+           value: '$.ref',
+           expressionType: 'JSONPath',
+           regexpFilter: 'refs/heads/', //Optional, defaults to empty string
+           defaultValue: ''  //Optional, defaults to empty string
+          ]
+         ],
+         causeString: 'Triggered on $ref',
+         token: 'build-kola-test-containers',
+         tokenCredentialId: '',
+         printContributedVariables: true,
+         printPostContent: true,
+         silentResponse: false,
+         regexpFilterText: '$COREOS_ASSEMBLER_GIT_REF',
+         regexpFilterExpression: 'main|rhcos-.*'
+        ]
+    ]),
+    parameters([
+      string(name: 'ARCHES',
+             description: 'Space-separated list of target architectures',
+             defaultValue: "x86_64" + " " + pipeutils.get_supported_additional_arches().join(" "),
+             trim: true),
+      string(name: 'COREOS_ASSEMBLER_GIT_URL',
+             description: 'Override the coreos-assembler git repo to use',
+             //defaultValue: "https://github.com/coreos/coreos-assembler.git",
+             defaultValue: "https://github.com/jbtrystram/coreos-assembler.git",
+             trim: true),
+      string(name: 'COREOS_ASSEMBLER_GIT_REF',
+             description: 'Override the coreos-assembler git ref to use',
+             //defaultValue: "main",
+             defaultValue: "coreos-owned-test-images",
+             trim: true),
+      string(name: 'CONTAINER_REGISTRY_ORG',
+             description: 'Override the registry org to push the containers to',
+             //defaultValue: "quay.io/coreos-assembler",
+             defaultValue: "quay.io/jbtrystramtestimages",
+             trim: true),
+      string(name: 'PATH_TO_CONTEXTS',
+             description: """Override the path to the contexts directories to use as build contexts.
+             Each directory should contain a containerfile and the image will be named after the directory name.""",
+             defaultValue: "tests/containers/",
+             trim: true)
+    ]),
+    buildDiscarder(logRotator(
+        numToKeepStr: '100',
+        artifactNumToKeepStr: '100'
+    )),
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+])
+
+node {
+    change = checkout(
+        changelog: true,
+        poll: false,
+        scm: [
+            $class: 'GitSCM',
+            branches: [[name: "origin/${params.COREOS_ASSEMBLER_GIT_REF}"]],
+            userRemoteConfigs: [[url: params.COREOS_ASSEMBLER_GIT_URL]],
+            extensions: [[$class: 'CloneOption',
+                          noTags: true,
+                          reference: '',
+                          shallow: true]]
+        ]
+    )
+
+    gitref = params.COREOS_ASSEMBLER_GIT_REF
+    def output = shwrapCapture("git rev-parse HEAD")
+    commit = output.substring(0,40)
+    shortcommit = commit.substring(0,7)
+}
+
+node {
+    change = checkout(
+        changelog: true,
+        poll: false,
+        scm: [
+            $class: 'GitSCM',
+            branches: [[name: "origin/${params.COREOS_ASSEMBLER_GIT_REF}"]],
+            userRemoteConfigs: [[url: params.COREOS_ASSEMBLER_GIT_URL]],
+            extensions: [[$class: 'CloneOption',
+                          noTags: true,
+                          reference: '',
+                          shallow: true]]
+        ]
+    )
+
+    gitref = params.COREOS_ASSEMBLER_GIT_REF
+    def output = shwrapCapture("git rev-parse HEAD")
+    commit = output.substring(0,40)
+    shortcommit = commit.substring(0,7)
+}
+
+node {
+    def path=params.PATH_TO_CONTEXTS
+    // gather the context folders list
+    contexts = sh(returnStdout: true, script: """
+        cd ${path}
+        find . -maxdepth 1 -mindepth 1 -type d -not -path '*/\\.*' -exec basename {} \\;
+    """).trim().split("\n")
+}
+
+currentBuild.description = "[${gitref}@${shortcommit}] Waiting"
+
+// Get the list of requested architectures to build for
+def basearches = params.ARCHES.split() as Set
+// and  the list of contexts to build
+def contexts = contexts as Set
+
+lock(resource: "build-containers") {
+    cosaPod(image: "quay.io/coreos-assembler/coreos-assembler",
+            memory: "512Mi", kvm: false,
+            serviceAccount: "jenkins") {
+    timeout(time: 60, unit: 'MINUTES') {
+    try {
+
+        currentBuild.description = "[${gitref}@${shortcommit}] Running"
+
+        // By default we will allow re-using cache layers for one day.
+        // This is mostly so we can prevent re-downloading the RPMS
+        // and repo metadata and over again in a given day for successive
+        // builds.
+        def cacheTTL = "24h"
+        def force = ""
+        if (params.FORCE) {
+            force = '--force'
+            // Also set cacheTTL to 0.1s to allow users an escape hatch
+            // to force no cache layer usage.
+            cacheTTL = "0.1s"
+        }
+
+        withCredentials([file(credentialsId: 'jbtrystram-testorg-robot-push-secret', variable: 'REGISTRY_SECRET')]) {
+            stage('Build Containers') {
+                parallel basearches.collectEntries{arch -> [arch, {
+                    for (directory in contexts) {
+                        def dir = params.PATH_TO_CONTEXTS + directory
+                        pipeutils.withPodmanRemoteArchBuilder(arch: arch) {
+                            shwrap("""
+                                cosa remote-build-container \
+                                    --git-sub-dir $dir\
+                                    --arch $arch --cache-ttl ${cacheTTL} \
+                                    --git-ref $commit ${force} \
+                                    --git-url ${params.COREOS_ASSEMBLER_GIT_URL} \
+                                    --repo ${params.CONTAINER_REGISTRY_ORG}/$directory \
+                                    --push-to-registry --auth=\$REGISTRY_SECRET
+                            """)
+                        }
+                }
+                }]}
+            }
+
+            stage('Push manifests') {
+                for (architecture in basearches) {
+                    def arch = architecture
+                    for (directory in contexts) {
+                        def dir = directory
+                        images += " --image=docker://${params.CONTAINER_REGISTRY_ORG}/${dir}:${arch}-${shortcommit}"
+                    }
+                }
+
+                for (directory in contexts) {
+                    def dir = directory
+                    shwrap("""
+                    export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507
+                    cosa push-container-manifest --v2s2 \
+                        --auth=\$REGISTRY_SECRET --tag ${gitref} \
+                        --repo ${params.CONTAINER_REGISTRY_ORG}/$dir ${images}
+                    """)
+                    // Specifically for the `main` branch let's also update the `latest` tag
+                    // If there was a way to alias/tie these two together in the Quay UI
+                    // that would be preferable.
+                    if (gitref == "main") {
+                        shwrap("""
+                        export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507
+                        skopeo copy --all --authfile \$REGISTRY_SECRET   \
+                            docker://${params.CONTAINER_REGISTRY_ORG}/$dir:main \
+                            docker://${params.CONTAINER_REGISTRY_ORG}/$dir:latest
+                        """)
+                    }
+                }
+            }
+
+
+            stage('Delete Intermediate Tags') {
+                for (directory in contexts) {
+                    def dir = directory
+                        shwrap("""
+                        export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507
+                        skopeo delete --authfile=\$REGISTRY_SECRET \
+                            docker://${params.CONTAINER_REGISTRY_ORG}/$dir:${arch}-${shortcommit}
+                        """)
+                }
+            }
+
+        currentBuild.result = 'SUCCESS'
+
+    } catch (e) {
+        currentBuild.result = 'FAILURE'
+        throw e
+    } finally {
+        if (currentBuild.result == 'SUCCESS') {
+            currentBuild.description = "[${gitref}@${shortcommit}] ⚡"
+        } else {
+            currentBuild.description = "[${gitref}@${shortcommit}] ❌"
+        }
+        if (currentBuild.result != 'SUCCESS') {
+            message = "build-kola-test-containers #${env.BUILD_NUMBER} <${env.BUILD_URL}|:jenkins:> <${env.RUN_DISPLAY_URL}|:ocean:> [${gitref}@${shortcommit}]"
+            pipeutils.trySlackSend(message: message)
+        }
+    }
+}}} // cosaPod, timeout, and lock finish here
+