From 3a687e7510b9f4c716f3ad7cc3a86492979a266d Mon Sep 17 00:00:00 2001 From: Hans Kratz Date: Sun, 31 Oct 2021 20:49:57 +0100 Subject: [PATCH 1/2] Use ubuntu image to download openssl, curl sources, cacert.pem --- .../host-x86_64/dist-i686-linux/Dockerfile | 27 +++++++++++----- .../host-x86_64/dist-x86_64-linux/Dockerfile | 27 +++++++++++----- .../dist-x86_64-linux/build-curl.sh | 13 ++------ .../dist-x86_64-linux/build-openssl.sh | 13 ++------ .../download-openssl-curl.sh | 10 ++++++ .../host-x86_64/shared/ISRG_Root_X1.crt | 31 ------------------- 6 files changed, 54 insertions(+), 67 deletions(-) create mode 100755 src/ci/docker/host-x86_64/dist-x86_64-linux/download-openssl-curl.sh delete mode 100644 src/ci/docker/host-x86_64/shared/ISRG_Root_X1.crt diff --git a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile index 63836654293f2..2b4b78e81350d 100644 --- a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile @@ -1,3 +1,15 @@ +# We need recent curl, OpenSSL and CA certificates, so we can download further +# dependencies in the debian:6 image. We use an ubuntu 20.04 image download +# those. +FROM ubuntu:20.04 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + ca-certificates +WORKDIR /tmp +COPY host-x86_64/dist-x86_64-linux/download-openssl-curl.sh /tmp/ +RUN ./download-openssl-curl.sh + # We use Debian 6 (glibc 2.11, kernel 2.6.32) as a common base for other # distros that still need Rust support: RHEL 6 (glibc 2.12, kernel 2.6.32) and # SLES 11 SP4 (glibc 2.11, kernel 3.0). @@ -14,8 +26,6 @@ RUN apt-get update && \ apt-get install --allow-unauthenticated -y --no-install-recommends \ automake \ bzip2 \ - ca-certificates \ - curl \ file \ g++ \ g++-multilib \ @@ -34,11 +44,6 @@ RUN apt-get update && \ xz-utils \ zlib1g-dev -# Install new Let's Encrypt root CA certificate and remove the expired one. -COPY host-x86_64/shared/ISRG_Root_X1.crt /usr/local/share/ca-certificates/ISRG_Root_X1.crt -RUN sed -i '/mozilla\/DST_Root_CA_X3\.crt/d' /etc/ca-certificates.conf -RUN /usr/sbin/update-ca-certificates - ENV PATH=/rustroot/bin:$PATH ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig @@ -50,6 +55,7 @@ COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/ # static.rust-lang.org. This'll be used to link into libcurl below (and used # later as well), so build a copy of OpenSSL with dynamic libraries into our # generic root. +COPY --from=0 /tmp/openssl.tar.gz /tmp/openssl.tar.gz COPY host-x86_64/dist-x86_64-linux/build-openssl.sh /tmp/ RUN ./build-openssl.sh @@ -59,8 +65,13 @@ RUN ./build-openssl.sh # # Note that we also disable a bunch of optional features of curl that we don't # really need. +COPY --from=0 /tmp/curl.tar.xz /tmp/curl.tar.xz COPY host-x86_64/dist-x86_64-linux/build-curl.sh /tmp/ -RUN ./build-curl.sh && apt-get remove -y curl +RUN ./build-curl.sh + +# Use up-to-date curl CA bundle +COPY --from=0 /tmp/cacert.pem /tmp/cacert.pem +ENV CURL_CA_BUNDLE /tmp/cacert.pem # binutils < 2.22 has a bug where the 32-bit executables it generates # immediately segfault in Rust, so we need to install our own binutils. diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 7b560aaaaa688..50452349931e8 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -1,3 +1,15 @@ +# We need recent curl, OpenSSL and CA certificates, so we can download further +# dependencies in the debian:6 image. We use an ubuntu 20.04 image download +# those. +FROM ubuntu:20.04 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + ca-certificates +WORKDIR /tmp +COPY host-x86_64/dist-x86_64-linux/download-openssl-curl.sh /tmp/ +RUN ./download-openssl-curl.sh + # We use Debian 6 (glibc 2.11, kernel 2.6.32) as a common base for other # distros that still need Rust support: RHEL 6 (glibc 2.12, kernel 2.6.32) and # SLES 11 SP4 (glibc 2.11, kernel 3.0). @@ -14,8 +26,6 @@ RUN apt-get update && \ apt-get install --allow-unauthenticated -y --no-install-recommends \ automake \ bzip2 \ - ca-certificates \ - curl \ file \ g++ \ g++-multilib \ @@ -34,11 +44,6 @@ RUN apt-get update && \ xz-utils \ zlib1g-dev -# Install new Let's Encrypt root CA certificate and remove the expired one. -COPY host-x86_64/shared/ISRG_Root_X1.crt /usr/local/share/ca-certificates/ISRG_Root_X1.crt -RUN sed -i '/mozilla\/DST_Root_CA_X3\.crt/d' /etc/ca-certificates.conf -RUN /usr/sbin/update-ca-certificates - ENV PATH=/rustroot/bin:$PATH ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig @@ -50,6 +55,7 @@ COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/ # static.rust-lang.org. This'll be used to link into libcurl below (and used # later as well), so build a copy of OpenSSL with dynamic libraries into our # generic root. +COPY --from=0 /tmp/openssl.tar.gz /tmp/openssl.tar.gz COPY host-x86_64/dist-x86_64-linux/build-openssl.sh /tmp/ RUN ./build-openssl.sh @@ -59,8 +65,13 @@ RUN ./build-openssl.sh # # Note that we also disable a bunch of optional features of curl that we don't # really need. +COPY --from=0 /tmp/curl.tar.xz /tmp/curl.tar.xz COPY host-x86_64/dist-x86_64-linux/build-curl.sh /tmp/ -RUN ./build-curl.sh && apt-get remove -y curl +RUN ./build-curl.sh + +# Use up-to-date curl CA bundle +COPY --from=0 /tmp/cacert.pem /tmp/cacert.pem +ENV CURL_CA_BUNDLE /tmp/cacert.pem # binutils < 2.22 has a bug where the 32-bit executables it generates # immediately segfault in Rust, so we need to install our own binutils. diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-curl.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-curl.sh index 6efa789756619..88ee96eaa89b5 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-curl.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-curl.sh @@ -3,18 +3,11 @@ set -ex source shared.sh -VERSION=7.66.0 - -# This needs to be downloaded directly from S3, it can't go through the CDN. -# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1 -# (without paying an absurd amount of money). -curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/curl-$VERSION.tar.xz \ - | xz --decompress \ - | tar xf - +tar xJf curl.tar.xz mkdir curl-build cd curl-build -hide_output ../curl-$VERSION/configure \ +hide_output ../curl-*/configure \ --prefix=/rustroot \ --with-ssl=/rustroot \ --disable-sspi \ @@ -35,4 +28,4 @@ hide_output make install cd .. rm -rf curl-build -rm -rf curl-$VERSION +rm -rf curl-* diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-openssl.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-openssl.sh index 34bbe19d2fac3..b48b5c4c00aae 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-openssl.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-openssl.sh @@ -3,21 +3,14 @@ set -ex source shared.sh -VERSION=1.0.2k +tar xzf openssl.tar.gz -# This needs to be downloaded directly from S3, it can't go through the CDN. -# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1 -# (without paying an absurd amount of money). -URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz - -curl $URL | tar xzf - - -cd openssl-$VERSION +cd openssl-* hide_output ./config --prefix=/rustroot shared -fPIC hide_output make -j$(nproc) hide_output make install cd .. -rm -rf openssl-$VERSION +rm -rf openssl-* # Make the system cert collection available to the new install. ln -nsf /etc/pki/tls/cert.pem /rustroot/ssl/ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/download-openssl-curl.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/download-openssl-curl.sh new file mode 100755 index 0000000000000..ca40a8cf7daa9 --- /dev/null +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/download-openssl-curl.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -ex + +OPENSSL_VERSION=1.0.2k +CURL_VERSION=7.66.0 + +curl -f https://ci-mirrors.rust-lang.org/rustc/openssl-$OPENSSL_VERSION.tar.gz -o openssl.tar.gz +curl -f https://ci-mirrors.rust-lang.org/rustc/curl-$CURL_VERSION.tar.xz -o curl.tar.xz +curl -f https://curl.se/ca/cacert.pem -o cacert.pem diff --git a/src/ci/docker/host-x86_64/shared/ISRG_Root_X1.crt b/src/ci/docker/host-x86_64/shared/ISRG_Root_X1.crt deleted file mode 100644 index b85c8037f6b60..0000000000000 --- a/src/ci/docker/host-x86_64/shared/ISRG_Root_X1.crt +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw -TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh -cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 -WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu -ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY -MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc -h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ -0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U -A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW -T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH -B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC -B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv -KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn -OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn -jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw -qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI -rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV -HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq -hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL -ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ -3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK -NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 -ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur -TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC -jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc -oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq -4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA -mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d -emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= ------END CERTIFICATE----- From aef51a0697b6906b0545b323426db05d189049fc Mon Sep 17 00:00:00 2001 From: Hans Kratz Date: Mon, 1 Nov 2021 18:32:24 +0100 Subject: [PATCH 2/2] Ignore files copied from previous stage when generating hash. --- src/ci/docker/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index b1dacf79d2695..93b4f435d4db7 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -50,7 +50,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then # Look for all source files involves in the COPY command copied_files=/tmp/.docker-copied-files.txt rm -f "$copied_files" - for i in $(sed -n -e 's/^COPY \(.*\) .*$/\1/p' "$docker_dir/$image/Dockerfile"); do + for i in $(sed -n -e '/^COPY --from=/! s/^COPY \(.*\) .*$/\1/p' \ + "$docker_dir/$image/Dockerfile"); do # List the file names find "$script_dir/$i" -type f >> $copied_files done