feat: support building centos/debian Docker images in multiplatform f…

…ormat (elastic#1091)

* chore: copy Centos+Systemd Dockerfile from origin

See https://github.com/CentOS/CentOS-Dockerfiles/tree/master/systemd/centos7

* chore: copy debian+systemd Dockerfile from origin

See https://github.com/alehaa/docker-debian-systemd

* chore: add script to build&push ARCH-based images for centos and debian

* chore: add script to push the multiplatform manifest for centos and debian

This script leverages infra's tool to write the manifest, which needs to
be ran right after the images have been built and pushed. Therefore, the
tool will write the manifest for both platforms (AMD/ARM), inspecting
the existing platform-specific repositories, combining them into the target.

FYI, the '-ARCH' placeholder will be replaced with the values in the
'--platforms' argument

* chore: add regular pipeline to build the docker images

* fix: default arch variable value

* chore: abstract image name from file system

* chore: couple agent's base box with stack platform

.ci/build-docker-images.groovy

@@ -0,0 +1,94 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'ubuntu-20' }
+  environment {
+    REPO = 'e2e-testing'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    HOME = "${env.WORKSPACE}"
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+    JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    cron 'H H(0-5) * * 1-5'
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}",
+          branch: "${params.BRANCH_REFERENCE}",
+          repo: "https://github.com/elastic/${REPO}.git",
+          credentialsId: "${JOB_GIT_CREDENTIALS}"
+        )
+        stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+      }
+    }
+    stage('Build AMD Docker images'){
+      agent { label 'ubuntu-20 && immutable && docker' }
+      environment {
+        HOME = "${env.WORKSPACE}/${BASE_DIR}"
+      }
+      steps {
+        deleteDir()
+        unstash 'source'
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+        dir("${BASE_DIR}") {
+          withEnv(["ARCH=amd64"]) {
+            sh(label: 'Build AMD images', script: '.ci/scripts/build-docker-images.sh')
+          }
+        }
+      }
+    }
+    stage('Build ARM Docker images'){
+      agent { label 'arm && immutable && docker' }
+      environment {
+        HOME = "${env.WORKSPACE}/${BASE_DIR}"
+      }
+      steps {
+        deleteDir()
+        unstash 'source'
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+        dir("${BASE_DIR}") {
+          withEnv(["ARCH=arm64"]) {
+            sh(label: 'Build ARM images', script: '.ci/scripts/build-docker-images.sh')
+          }
+        }
+      }
+    }
+    stage('Push multiplatform manifest'){
+      agent { label 'ubuntu-20 && immutable && docker' }
+      environment {
+        HOME = "${env.WORKSPACE}/${BASE_DIR}"
+      }
+      steps {
+        deleteDir()
+        unstash 'source'
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+        dir("${BASE_DIR}") {
+          sh(label: 'Push multiplatform manifest', script: '.ci/scripts/push-multiplatform-manifest.sh')
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}

.ci/docker/centos-systemd/Dockerfile

@@ -0,0 +1,18 @@
+FROM centos:7
+
+ENV container docker
+
+LABEL maintainer="manuel.delapena@elastic.co"
+
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+rm -f /lib/systemd/system/multi-user.target.wants/*;\
+rm -f /etc/systemd/system/*.wants/*;\
+rm -f /lib/systemd/system/local-fs.target.wants/*; \
+rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+rm -f /lib/systemd/system/basic.target.wants/*;\
+rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+VOLUME [ "/sys/fs/cgroup" ]
+
+CMD ["/usr/sbin/init"]

.ci/docker/debian-systemd/Dockerfile

@@ -0,0 +1,141 @@
+# This file is part of docker-debian-systemd.
+#
+# Copyright (c)
+#   2018-2019 Alexander Haase <ahaase@alexhaase.de>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# This image bases on the regular Debian image. By default the 'latest' tag
+# (pointing to the current stable release) of the parent image will be used.
+# However, an alternate parent tag may be set by defining the 'TAG' build
+# argument to a specific Debian release, e.g. 'stretch' or 'buster'.
+ARG  TAG=latest
+FROM debian:${TAG}
+LABEL maintainer="manuel.delapena@elastic.co"
+
+# Configure the debconf frontend.
+#
+# This image doesn't include whiptail, dialog, nor the readline perl module.
+# Therefore, the debconf frontend will be set to 'teletype' to avoid error
+# messages about no dialog frontend could be found.
+RUN echo 'debconf debconf/frontend select teletype' | debconf-set-selections
+
+
+# Install the necessary packages.
+#
+# In addition to the regular Debian base image, a BASIC set of packages from the
+# Debian minimal configuration will be installed. After all packages have been
+# installed, the apt caches and some log files will be removed to minimize the
+# image.
+#
+# NOTE: An upgrade will be performed to include updates and security fixes of
+#       installed packages that received updates in the Debian repository after
+#       the upstream image has been created.
+#
+# NOTE: No syslog daemon will be installed, as systemd's journald should fit
+#       most needs. Please file an issue if you think this should be changed.
+RUN apt-get update
+RUN apt-get dist-upgrade -y
+RUN apt-get install -y --no-install-recommends \
+        systemd      \
+        systemd-sysv \
+        cron         \
+        anacron
+
+RUN apt-get clean
+RUN rm -rf                        \
+    /var/lib/apt/lists/*          \
+    /var/log/alternatives.log     \
+    /var/log/apt/history.log      \
+    /var/log/apt/term.log         \
+    /var/log/dpkg.log
+
+
+# Configure systemd.
+#
+# For running systemd inside a Docker container, some additional tweaks are
+# required. For a detailed list see:
+#
+# https://developers.redhat.com/blog/2016/09/13/ \
+#   running-systemd-in-a-non-privileged-container/
+#
+# Additional tweaks will be applied in the final image below.
+
+# To avoid ugly warnings when running this image on a host running systemd, the
+# following units will be masked.
+#
+# NOTE: This will not remove ALL warnings in all Debian releases, but seems to
+#       work for stretch.
+RUN systemctl mask --   \
+    dev-hugepages.mount \
+    sys-fs-fuse-connections.mount
+
+# The machine-id should be generated when creating the container. This will be
+# done automatically if the file is not present, so let's delete it.
+RUN rm -f           \
+    /etc/machine-id \
+    /var/lib/dbus/machine-id
+
+
+
+
+# Build the final image.
+#
+# To get a minimal image without deleted files in intermediate layers, the
+# contents of the image previously built will be copied into a second version of
+# the parent image.
+#
+# NOTE: This method requires buildkit, as the differ of buildkit will copy
+#       changed files only and we'll get a minimal image with just the changed
+#       files in a single new layer.
+#
+# NOTE: All settings related to the image's environment (e.g. CMD, ENV and
+#       VOLUME settings) need to be set in the following image definition to be
+#       used by child images and containers.
+
+FROM debian:${TAG}
+COPY --from=0 / /
+
+
+# Configure systemd.
+#
+# For running systemd inside a Docker container, some additional tweaks are
+# required. Some of them have already been applied above.
+#
+# The 'container' environment variable tells systemd that it's running inside a
+# Docker container environment.
+ENV container docker
+
+# A different stop signal is required, so systemd will initiate a shutdown when
+# running 'docker stop <container>'.
+STOPSIGNAL SIGRTMIN+3
+
+# The host's cgroup filesystem need's to be mounted (read-only) in the
+# container. '/run', '/run/lock' and '/tmp' need to be tmpfs filesystems when
+# running the container without 'CAP_SYS_ADMIN'.
+#
+# NOTE: For running Debian stretch, 'CAP_SYS_ADMIN' still needs to be added, as
+#       stretch's version of systemd is not recent enough. Buster will run just
+#       fine without 'CAP_SYS_ADMIN'.
+VOLUME [ "/sys/fs/cgroup", "/run", "/run/lock", "/tmp" ]
+
+# As this image should run systemd, the default command will be changed to start
+# the init system. CMD will be preferred in favor of ENTRYPOINT, so one may
+# override it when creating the container to e.g. to run a bash console instead.
+CMD [ "/sbin/init" ]

.ci/jobs/build-docker-images.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: Beats/build-docker-images
+    display-name: E2E Tests Docker images
+    description: Job to pre-build docker images used in E2E tests.
+    view: Beats
+    project-type: pipeline
+    parameters:
+      - string:
+          name: BRANCH_REFERENCE
+          default: master
+          description: the Git branch specifier
+    pipeline-scm:
+      script-path: .ci/build-docker-images.groovy
+      scm:
+        - git:
+            url: git@github.com:elastic/e2e-testint.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: true
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/e2e-testing.git
+            branches:
+              - $BRANCH_REFERENCE
+    triggers:
+      - timed: 'H H(0-5) * * 1-5'

.ci/scripts/build-docker-images.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+## Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+## or more contributor license agreements. Licensed under the Elastic License;
+## you may not use this file except in compliance with the Elastic License.
+
+set -euxo pipefail
+
+ARCH="${ARCH:-amd64}"
+
+readonly ELASTIC_REGISTRY="docker.elastic.co"
+readonly OBSERVABILITY_CI_REGISTRY="${ELASTIC_REGISTRY}/observability-ci"
+
+main() {
+  _build_and_push "centos-systemd"
+  _build_and_push "debian-systemd"
+}
+
+_build_and_push() {
+  local image="${1}"
+
+  local platformSpecificImage="${OBSERVABILITY_CI_REGISTRY}/${image}-${ARCH}:latest"
+
+  docker build -t ${platformSpecificImage} .ci/docker/${image}
+
+  docker push ${platformSpecificImage}
+}
+
+main "$@"

.ci/scripts/push-multiplatform-manifest.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+## Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+## or more contributor license agreements. Licensed under the Elastic License;
+## you may not use this file except in compliance with the Elastic License.
+
+set -euxo pipefail
+
+readonly ELASTIC_REGISTRY="docker.elastic.co"
+readonly MANIFEST_TOOL_IMAGE="${ELASTIC_REGISTRY}/infra/manifest-tool:latest"
+readonly OBSERVABILITY_CI_REGISTRY="${ELASTIC_REGISTRY}/observability-ci"
+
+main() {
+  _push_multiplatform_manifest "centos-systemd"
+  _push_multiplatform_manifest "debian-systemd"
+}
+
+_push_multiplatform_manifest() {
+  local image="${1}"
+
+  local fqn="${OBSERVABILITY_CI_REGISTRY}/${image}:latest"
+  # the '-ARCH' placeholder will be replaced with the values in the '--platforms' argument
+  local templateFqn="${OBSERVABILITY_CI_REGISTRY}/${image}-ARCH:latest"
+
+  docker run --rm \
+    --mount src=${HOME}/.docker,target=/docker-config,type=bind \
+      ${MANIFEST_TOOL_IMAGE} --docker-cfg "/docker-config" \
+        push from-args \
+          --platforms linux/amd64,linux/arm64 \
+          --template ${templateFqn} \
+          --target ${fqn}
+}
+
+main "$@"

cli/config/compose/services/centos-systemd/docker-compose.yml

@@ -4,6 +4,7 @@ services:
     image: centos/systemd:${centos_systemdTag:-latest} 
     container_name: ${centos_systemdContainerName}
     entrypoint: "/usr/sbin/init"
+    platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
       - ${centos_systemdAgentBinarySrcPath:-.}:${centos_systemdAgentBinaryTargetPath:-/tmp}

cli/config/compose/services/debian-systemd/docker-compose.yml

@@ -4,6 +4,7 @@ services:
     image: alehaa/debian-systemd:${debian_systemdTag:-stretch}
     container_name: ${debian_systemdContainerName} 
     entrypoint: "/sbin/init"
+    platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
       - ${debian_systemdAgentBinarySrcPath:-.}:${debian_systemdAgentBinaryTargetPath:-/tmp}