diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index a28cb24a8166f..2b34142b3b0f3 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -832,17 +832,6 @@ impl Build {
         if target.contains("apple-darwin") {
             base.push("-stdlib=libc++".into());
         }
-        // This is a hack, because newer binutils broke things on some vms/distros
-        // (i.e., linking against unknown relocs disabled by the following flag)
-        // See: https://github.com/rust-lang/rust/issues/34978
-        match target {
-            "i586-unknown-linux-gnu" |
-            "i686-unknown-linux-musl" |
-            "x86_64-unknown-linux-musl" => {
-                base.push("-Wa,-mrelax-relocations=no".into());
-            },
-            _ => {},
-        }
         return base
     }
 
diff --git a/src/ci/docker/linux-tested-targets/Dockerfile b/src/ci/docker/linux-tested-targets/Dockerfile
index 9906b162cc24a..e935226e361fb 100644
--- a/src/ci/docker/linux-tested-targets/Dockerfile
+++ b/src/ci/docker/linux-tested-targets/Dockerfile
@@ -34,6 +34,14 @@ ENV RUST_CONFIGURE_ARGS \
       --musl-root-x86_64=/musl-x86_64 \
       --musl-root-i686=/musl-i686
 
+# Newer binutils broke things on some vms/distros (i.e., linking against
+# unknown relocs disabled by the following flag), so we need to go out of our
+# way to produce "super compatible" binaries.
+#
+# See: https://github.com/rust-lang/rust/issues/34978
+ENV CFLAGS_i686_unknown_linux_gnu=-Wa,-mrelax-relocations=no \
+    CFLAGS_x86_64_unknown_linux_gnu=-Wa,-mrelax-relocations=no
+
 ENV SCRIPT \
       python2.7 ../x.py test \
           --target x86_64-unknown-linux-musl \
diff --git a/src/ci/docker/linux-tested-targets/build-musl.sh b/src/ci/docker/linux-tested-targets/build-musl.sh
index 260dc11509f47..ce62c392241d8 100644
--- a/src/ci/docker/linux-tested-targets/build-musl.sh
+++ b/src/ci/docker/linux-tested-targets/build-musl.sh
@@ -11,7 +11,10 @@
 
 set -ex
 
-export CFLAGS="-fPIC"
+# We need to mitigate rust-lang/rust#34978 when compiling musl itself as well
+export CFLAGS="-fPIC -Wa,-mrelax-relocations=no"
+export CXXFLAGS="-Wa,-mrelax-relocations=no"
+
 MUSL=musl-1.1.14
 curl https://www.musl-libc.org/releases/$MUSL.tar.gz | tar xzf -
 cd $MUSL