Merge #982

982: Add concept of a native Dockerfile. r=Emilgardis a=Alexhuszagh



Co-authored-by: Alex Huszagh <ahuszagh@gmail.com>

.changes/982.json

@@ -0,0 +1,4 @@
+{
+    "type": "internal",
+    "description": "use generic dockerfiles for when the toolchain and image platfom match."
+}

docker/Dockerfile.native

@@ -0,0 +1,31 @@
+# This dockerfile is used when the target matches the images platform in `build-docker-image`
+FROM ubuntu:20.04
+ARG DEBIAN_FRONTEND=noninteractive
+
+COPY common.sh lib.sh /
+RUN /common.sh
+
+COPY cmake.sh /
+RUN /cmake.sh
+
+COPY xargo.sh /
+RUN /xargo.sh
+
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG CROSS_TARGET_TRIPLE
+
+COPY qemu.sh native-qemu.sh /
+RUN /native-qemu.sh
+
+COPY dropbear.sh /
+RUN /dropbear.sh
+
+COPY linux-image.sh native-linux-image.sh /
+RUN /native-linux-image.sh
+
+COPY linux-runner native-linux-runner base-runner.sh /
+
+ENV CROSS_TARGETARCH=$TARGETARCH
+ENV CROSS_TARGETVARIANT=$TARGETVARIANT
+ENV CARGO_TARGET_${CROSS_TARGET_TRIPLE}_RUNNER="/native-linux-runner"

docker/Dockerfile.native.centos

@@ -0,0 +1,40 @@
+FROM ubuntu:20.04
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG CROSS_TARGET_TRIPLE
+
+COPY lib.sh /
+COPY linux-image.sh native-linux-image.sh /
+RUN /native-linux-image.sh
+
+FROM centos:7
+
+COPY common.sh lib.sh /
+RUN /common.sh
+
+COPY cmake.sh /
+RUN /cmake.sh
+
+COPY xargo.sh /
+RUN /xargo.sh
+
+# these need to be present in **both** FROM sections
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG CROSS_TARGET_TRIPLE
+
+COPY qemu.sh native-qemu.sh /
+RUN /native-qemu.sh
+
+COPY dropbear.sh /
+RUN /dropbear.sh
+
+COPY --from=0 /qemu /qemu
+
+COPY linux-runner native-linux-runner base-runner.sh /
+
+ENV CROSS_TARGETARCH=$TARGETARCH
+ENV CROSS_TARGETVARIANT=$TARGETVARIANT
+ENV CARGO_TARGET_${CROSS_TARGET_TRIPLE}_RUNNER="/native-linux-runner"

docker/Dockerfile.x86_64-unknown-linux-gnu

@@ -10,6 +10,15 @@ RUN /cmake.sh
 COPY xargo.sh /
 RUN /xargo.sh
 
+RUN apt-get update && apt-get install --assume-yes --no-install-recommends \
+    g++-x86-64-linux-gnu \
+    libc6-dev-amd64-cross
+
+COPY deny-debian-packages.sh /
+RUN TARGET_ARCH=amd64 /deny-debian-packages.sh \
+    binutils \
+    binutils-x86-64-linux-gnu
+
 COPY qemu.sh /
 RUN /qemu.sh x86_64 softmmu
 
@@ -21,4 +30,11 @@ RUN /linux-image.sh x86_64
 
 COPY linux-runner base-runner.sh /
 
-ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="/linux-runner x86_64"
+ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \
+    CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="/linux-runner x86_64" \
+    CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc \
+    CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ \
+    BINDGEN_EXTRA_CLANG_ARGS_x86_64_unknown_linux_gnu="--sysroot=/usr/x86_64-linux-gnu" \
+    QEMU_LD_PREFIX=/usr/x86_64-linux-gnu \
+    RUST_TEST_THREADS=1 \
+    PKG_CONFIG_PATH="/usr/lib/x86_64-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}"

docker/lib.sh

@@ -86,3 +86,72 @@ download_gcc() {
 
     download_mirrors "gcc/gcc-${version}" "${filename}" "${GNU_MIRRORS[@]}"
 }
+
+docker_to_qemu_arch() {
+    local arch="${1}"
+    case "${arch}" in
+        arm64)
+            echo "aarch64"
+            ;;
+        386)
+            echo "i386"
+            ;;
+        amd64)
+            echo "x86_64"
+            ;;
+        arm|ppc64le|riscv64|s390x)
+            echo "${arch}"
+            ;;
+        *)
+            echo "Unknown Docker image architecture, got \"${arch}\"." >&2
+            exit 1
+            ;;
+    esac
+}
+
+docker_to_linux_arch() {
+    # variant may not be provided
+    local oldstate
+    oldstate="$(set +o)"
+    set +u
+
+    local arch="${1}"
+    local variant="${2}"
+    case "${arch}" in
+        arm64)
+            echo "aarch64"
+            ;;
+        386)
+            echo "i686"
+            ;;
+        amd64)
+            echo "x86_64"
+            ;;
+        ppc64le)
+            echo "powerpc64le"
+            ;;
+        arm)
+            case "${variant}" in
+                v6)
+                    echo "arm"
+                    ;;
+                ""|v7)
+                    echo "armv7"
+                    ;;
+                *)
+                    echo "Unknown Docker image variant, got \"${variant}\"." >&2
+                    exit 1
+                    ;;
+            esac
+            ;;
+        riscv64|s390x)
+            echo "${arch}"
+            ;;
+        *)
+            echo "Unknown Docker image architecture, got \"${arch}\"." >&2
+            exit 1
+            ;;
+    esac
+
+    eval "${oldstate}"
+}

docker/linux-image.sh

@@ -171,12 +171,14 @@ main() {
         sharutils \
         gnupg
 
-    # amd64 has conflicting versions of the packages installed, so
+    # conflicting versions of some packages will be installed already for the host platform,
     # we need to remove the system installs later. since apt relies
     # on these packages, we need to download them and reinstall
     # using dpkg later, since we cannot redownload via apt.
+    local dpkg_arch
+    dpkg_arch=$(dpkg --print-architecture)
     local libgcc_packages=("${libgcc}:${arch}" "libstdc++6:${arch}")
-    if [[ "${arch}" == "amd64" ]]; then
+    if [[ "${arch}" == "${dpkg_arch}" ]]; then
         local libgcc_root=/qemu/libgcc
         mkdir -p "${libgcc_root}"
         pushd "${libgcc_root}"
@@ -186,6 +188,7 @@ main() {
 
     # Download packages
     mv /etc/apt/sources.list /etc/apt/sources.list.bak
+    mv /etc/apt/sources.list.d /etc/apt/sources.list.d.bak
     echo -e "${debsource}" > /etc/apt/sources.list
 
     # Old ubuntu does not support --add-architecture, so we directly change multiarch file
@@ -240,10 +243,10 @@ main() {
         ncurses-base"${ncurses}" \
         "zlib1g:${arch}"
 
-    if [[ "${arch}" != "amd64" ]]; then
+    if [[ "${arch}" != "${dpkg_arch}" ]]; then
         apt-get -d --no-install-recommends download "${libgcc_packages[@]}"
     else
-        # amd64 has conflicting versions of the packages installed
+        # host arch has conflicting versions of the packages installed
         # this prevents us from downloading them, so we need to
         # simply grab the last version from the debian sources.
         # we're search for a paragraph with:
@@ -380,7 +383,7 @@ EOF
     find . | cpio --create --format='newc' --quiet | gzip > ../initrd.gz
     cd -
 
-    if [[ "${arch}" == "amd64" ]]; then
+    if [[ "${arch}" == "${dpkg_arch}" ]]; then
         # need to reinstall these packages, since basic utilities rely on them.
         pushd "${libgcc_root}"
         dpkg -i --force-depends "${libgcc_root}"/*.deb
@@ -391,15 +394,16 @@ EOF
     # Clean up
     rm -rf "/qemu/${root}" "/qemu/${arch}"
     mv -f /etc/apt/sources.list.bak /etc/apt/sources.list
+    mv -f /etc/apt/sources.list.d.bak /etc/apt/sources.list.d
     if [ -f /etc/dpkg/dpkg.cfg.d/multiarch.bak ]; then
         mv /etc/dpkg/dpkg.cfg.d/multiarch.bak /etc/dpkg/dpkg.cfg.d/multiarch
     fi
-    # can fail if arch is used (amd64 and/or i386)
+    # can fail if arch is used (image arch, such as amd64 and/or i386)
     dpkg --remove-architecture "${arch}" || true
     apt-get update
 
     # need to reinstall the removed libgcc packages, which are required for apt
-    if [[ "${arch}" == "amd64" ]]; then
+    if [[ "${arch}" == "${dpkg_arch}" ]]; then
         apt-get install --no-install-recommends --assume-yes "${packages[@]}"
     fi
 

docker/linux-runner

@@ -16,7 +16,7 @@ fi
 arch="${1}"
 shift
 
-if [ "${CROSS_RUNNER}" = "" ]; then
+if [[ -z "${CROSS_RUNNER}" ]]; then
     if is_native_binary "${arch}"; then
         CROSS_RUNNER=native
     else

docker/native-linux-image.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -x
+set -eo pipefail
+
+# shellcheck disable=SC1091
+. lib.sh
+
+main() {
+    local arch
+    arch=$(docker_to_linux_arch "${TARGETARCH}" "${TARGETVARIANT}")
+    /linux-image.sh "${arch}"
+    rm "${0}"
+}
+
+main "${@}"

docker/native-linux-runner

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# shellcheck disable=SC1091
+. /lib.sh
+
+main() {
+    local arch
+    arch=$(docker_to_linux_arch "${CROSS_TARGETARCH}" "${CROSS_TARGETVARIANT}")
+
+    if [[ -z "${CROSS_RUNNER}" ]]; then
+        export CROSS_RUNNER=native
+    fi
+
+    exec /linux-runner "${arch}" "${@}"
+}
+
+main "${@}"

docker/native-qemu.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -x
+set -euo pipefail
+
+# shellcheck disable=SC1091
+. lib.sh
+
+main() {
+    local arch
+    arch=$(docker_to_qemu_arch "${TARGETARCH}")
+    /qemu.sh "${arch}" softmmu
+    rm "${0}"
+}
+
+main "${@}"

docker/qemu-runner

@@ -16,7 +16,7 @@ fi
 arch="${1}"
 shift
 
-if [ "${CROSS_RUNNER}" = "" ]; then
+if [[ -z "${CROSS_RUNNER}" ]]; then
     if is_native_binary "${arch}"; then
         CROSS_RUNNER=native
     else