feat: support building centos/debian Docker images in multiplatform format

.ci/build-docker-images.groovy

@@ -0,0 +1,94 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'ubuntu-20' }
+  environment {
+    REPO = 'e2e-testing'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    HOME = "${env.WORKSPACE}"
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+    JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    cron 'H H(0-5) * * 1-5'
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}",
+          branch: "${params.BRANCH_REFERENCE}",
+          repo: "https://github.com/elastic/${REPO}.git",
+          credentialsId: "${JOB_GIT_CREDENTIALS}"
+        )
+        stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+      }
+    }
+    stage('Build AMD Docker images'){
+      agent { label 'ubuntu-20 && immutable && docker' }
+      environment {
+        HOME = "${env.WORKSPACE}/${BASE_DIR}"
+      }
+      steps {
+        deleteDir()
+        unstash 'source'
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+        dir("${BASE_DIR}") {
+          withEnv(["ARCH=amd64"]) {
+            sh(label: 'Build AMD images', script: '.ci/scripts/build-docker-images.sh')
+          }
+        }
+      }
+    }
+    stage('Build ARM Docker images'){
+      agent { label 'arm && immutable && docker' }
+      environment {
+        HOME = "${env.WORKSPACE}/${BASE_DIR}"
+      }
+      steps {
+        deleteDir()
+        unstash 'source'
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+        dir("${BASE_DIR}") {
+          withEnv(["ARCH=arm64"]) {
+            sh(label: 'Build ARM images', script: '.ci/scripts/build-docker-images.sh')
+          }
+        }
+      }
+    }
+    stage('Push multiplatform manifest'){
+      agent { label 'ubuntu-20 && immutable && docker' }
+      environment {
+        HOME = "${env.WORKSPACE}/${BASE_DIR}"
+      }
+      steps {
+        deleteDir()
+        unstash 'source'
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+        dir("${BASE_DIR}") {
+          sh(label: 'Push multiplatform manifest', script: '.ci/scripts/push-multiplatform-manifest.sh')
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}

.ci/docker/centos-systemd/Dockerfile

@@ -0,0 +1,18 @@
+FROM centos:7
+
+ENV container docker
+
+LABEL maintainer="manuel.delapena@elastic.co"
+
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+rm -f /lib/systemd/system/multi-user.target.wants/*;\
+rm -f /etc/systemd/system/*.wants/*;\
+rm -f /lib/systemd/system/local-fs.target.wants/*; \
+rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+rm -f /lib/systemd/system/basic.target.wants/*;\
+rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+VOLUME [ "/sys/fs/cgroup" ]
+
+CMD ["/usr/sbin/init"]

.ci/docker/debian-systemd/Dockerfile

@@ -0,0 +1,141 @@
+# This file is part of docker-debian-systemd.
+#
+# Copyright (c)
+#   2018-2019 Alexander Haase <ahaase@alexhaase.de>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# This image bases on the regular Debian image. By default the 'latest' tag
+# (pointing to the current stable release) of the parent image will be used.
+# However, an alternate parent tag may be set by defining the 'TAG' build
+# argument to a specific Debian release, e.g. 'stretch' or 'buster'.
+ARG  TAG=latest
+FROM debian:${TAG}
+LABEL maintainer="manuel.delapena@elastic.co"
+
+# Configure the debconf frontend.
+#
+# This image doesn't include whiptail, dialog, nor the readline perl module.
+# Therefore, the debconf frontend will be set to 'teletype' to avoid error
+# messages about no dialog frontend could be found.
+RUN echo 'debconf debconf/frontend select teletype' | debconf-set-selections
+
+
+# Install the necessary packages.
+#
+# In addition to the regular Debian base image, a BASIC set of packages from the
+# Debian minimal configuration will be installed. After all packages have been
+# installed, the apt caches and some log files will be removed to minimize the
+# image.
+#
+# NOTE: An upgrade will be performed to include updates and security fixes of
+#       installed packages that received updates in the Debian repository after
+#       the upstream image has been created.
+#
+# NOTE: No syslog daemon will be installed, as systemd's journald should fit
+#       most needs. Please file an issue if you think this should be changed.
+RUN apt-get update
+RUN apt-get dist-upgrade -y
+RUN apt-get install -y --no-install-recommends \
+        systemd      \
+        systemd-sysv \
+        cron         \
+        anacron
+
+RUN apt-get clean
+RUN rm -rf                        \
+    /var/lib/apt/lists/*          \
+    /var/log/alternatives.log     \
+    /var/log/apt/history.log      \
+    /var/log/apt/term.log         \
+    /var/log/dpkg.log
+
+
+# Configure systemd.
+#
+# For running systemd inside a Docker container, some additional tweaks are
+# required. For a detailed list see:
+#
+# https://developers.redhat.com/blog/2016/09/13/ \
+#   running-systemd-in-a-non-privileged-container/
+#
+# Additional tweaks will be applied in the final image below.
+
+# To avoid ugly warnings when running this image on a host running systemd, the
+# following units will be masked.
+#
+# NOTE: This will not remove ALL warnings in all Debian releases, but seems to
+#       work for stretch.
+RUN systemctl mask --   \
+    dev-hugepages.mount \
+    sys-fs-fuse-connections.mount
+
+# The machine-id should be generated when creating the container. This will be
+# done automatically if the file is not present, so let's delete it.
+RUN rm -f           \
+    /etc/machine-id \
+    /var/lib/dbus/machine-id
+
+
+
+
+# Build the final image.
+#
+# To get a minimal image without deleted files in intermediate layers, the
+# contents of the image previously built will be copied into a second version of
+# the parent image.
+#
+# NOTE: This method requires buildkit, as the differ of buildkit will copy
+#       changed files only and we'll get a minimal image with just the changed
+#       files in a single new layer.
+#
+# NOTE: All settings related to the image's environment (e.g. CMD, ENV and
+#       VOLUME settings) need to be set in the following image definition to be
+#       used by child images and containers.
+
+FROM debian:${TAG}
+COPY --from=0 / /
+
+
+# Configure systemd.
+#
+# For running systemd inside a Docker container, some additional tweaks are
+# required. Some of them have already been applied above.
+#
+# The 'container' environment variable tells systemd that it's running inside a
+# Docker container environment.
+ENV container docker
+
+# A different stop signal is required, so systemd will initiate a shutdown when
+# running 'docker stop <container>'.
+STOPSIGNAL SIGRTMIN+3
+
+# The host's cgroup filesystem need's to be mounted (read-only) in the
+# container. '/run', '/run/lock' and '/tmp' need to be tmpfs filesystems when
+# running the container without 'CAP_SYS_ADMIN'.
+#
+# NOTE: For running Debian stretch, 'CAP_SYS_ADMIN' still needs to be added, as
+#       stretch's version of systemd is not recent enough. Buster will run just
+#       fine without 'CAP_SYS_ADMIN'.
+VOLUME [ "/sys/fs/cgroup", "/run", "/run/lock", "/tmp" ]
+
+# As this image should run systemd, the default command will be changed to start
+# the init system. CMD will be preferred in favor of ENTRYPOINT, so one may
+# override it when creating the container to e.g. to run a bash console instead.
+CMD [ "/sbin/init" ]

.ci/jobs/build-docker-images.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: Beats/build-docker-images
+    display-name: E2E Tests Docker images
+    description: Job to pre-build docker images used in E2E tests.
+    view: Beats
+    project-type: pipeline
+    parameters:
+      - string:
+          name: BRANCH_REFERENCE
+          default: master
+          description: the Git branch specifier
+    pipeline-scm:
+      script-path: .ci/build-docker-images.groovy
+      scm:
+        - git:
+            url: git@github.com:elastic/e2e-testint.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: true
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/e2e-testing.git
+            branches:
+              - $BRANCH_REFERENCE
+    triggers:
+      - timed: 'H H(0-5) * * 1-5'

.ci/scripts/build-docker-images.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+## Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+## or more contributor license agreements. Licensed under the Elastic License;
+## you may not use this file except in compliance with the Elastic License.
+
+set -euxo pipefail
+
+ARCH="${ARCH:-amd64}"
+
+readonly ELASTIC_REGISTRY="docker.elastic.co"
+readonly OBSERVABILITY_CI_REGISTRY="${ELASTIC_REGISTRY}/observability-ci"
+
+main() {
+  _build_and_push "centos-systemd"
+  _build_and_push "debian-systemd"
+}
+
+_build_and_push() {
+  local image="${1}"
+
+  local platformSpecificImage="${OBSERVABILITY_CI_REGISTRY}/${image}-${ARCH}:latest"
+
+  docker build -t ${platformSpecificImage} .ci/docker/${image}
+
+  docker push ${platformSpecificImage}
+}
+
+main "$@"

.ci/scripts/push-multiplatform-manifest.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+## Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+## or more contributor license agreements. Licensed under the Elastic License;
+## you may not use this file except in compliance with the Elastic License.
+
+set -euxo pipefail
+
+readonly ELASTIC_REGISTRY="docker.elastic.co"
+readonly MANIFEST_TOOL_IMAGE="${ELASTIC_REGISTRY}/infra/manifest-tool:latest"
+readonly OBSERVABILITY_CI_REGISTRY="${ELASTIC_REGISTRY}/observability-ci"
+
+main() {
+  _push_multiplatform_manifest "centos-systemd"
+  _push_multiplatform_manifest "debian-systemd"
+}
+
+_push_multiplatform_manifest() {
+  local image="${1}"
+
+  local fqn="${OBSERVABILITY_CI_REGISTRY}/${image}:latest"
+  # the '-ARCH' placeholder will be replaced with the values in the '--platforms' argument
+  local templateFqn="${OBSERVABILITY_CI_REGISTRY}/${image}-ARCH:latest"
+
+  docker run --rm \
+    --mount src=${HOME}/.docker,target=/docker-config,type=bind \
+      ${MANIFEST_TOOL_IMAGE} --docker-cfg "/docker-config" \
+        push from-args \
+          --platforms linux/amd64,linux/arm64 \
+          --template ${templateFqn} \
+          --target ${fqn}
+}
+
+main "$@"

cli/config/compose/services/centos-systemd/docker-compose.yml

@@ -4,6 +4,7 @@ services:
     image: centos/systemd:${centos_systemdTag:-latest} 
     container_name: ${centos_systemdContainerName}
     entrypoint: "/usr/sbin/init"
+    platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
       - ${centos_systemdAgentBinarySrcPath:-.}:${centos_systemdAgentBinaryTargetPath:-/tmp}

cli/config/compose/services/debian-systemd/docker-compose.yml

@@ -4,6 +4,7 @@ services:
     image: alehaa/debian-systemd:${debian_systemdTag:-stretch}
     container_name: ${debian_systemdContainerName} 
     entrypoint: "/sbin/init"
+    platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
       - ${debian_systemdAgentBinarySrcPath:-.}:${debian_systemdAgentBinaryTargetPath:-/tmp}