Auto merge of #65241 - tmiasko:no-std-san, r=alexcrichton

build-std compatible sanitizer support

### Motivation

When using `-Z sanitizer=*` feature it is essential that both user code and
standard library is instrumented. Otherwise the utility of sanitizer will be
limited, or its use will be impractical like in the case of memory sanitizer.

The recently introduced cargo feature build-std makes it possible to rebuild
standard library with arbitrary rustc flags. Unfortunately, those changes alone
do not make it easy to rebuild standard library with sanitizers, since runtimes
are dependencies of std that have to be build in specific environment,
generally not available outside rustbuild process. Additionally rebuilding them
requires presence of llvm-config and compiler-rt sources.

The goal of changes proposed here is to make it possible to avoid rebuilding
sanitizer runtimes when rebuilding the std, thus making it possible to
instrument standard library for use with sanitizer with simple, although
verbose command:

```
env CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS=-Zsanitizer=thread cargo test -Zbuild-std --target x86_64-unknown-linux-gnu
```

### Implementation

* Sanitizer runtimes are no long packed into crates. Instead, libraries build
  from compiler-rt are used as is, after renaming them into `librusc_rt.*`.
* rustc obtains runtimes from target libdir for default sysroot, so that
  they are not required in custom build sysroots created with build-std.
* The runtimes are only linked-in into executables to address issue #64629.
  (in previous design it was hard to avoid linking runtimes into static
  libraries produced by rustc as demonstrated by sanitizer-staticlib-link
  test, which still passes despite changes made in #64780).

cc @kennytm, @japaric, @Firstyear, @choller

Cargo.lock

@@ -3345,17 +3345,6 @@ dependencies = [
  "smallvec 1.0.0",
 ]
 
-[[package]]
-name = "rustc_asan"
-version = "0.0.0"
-dependencies = [
- "alloc",
- "build_helper",
- "cmake",
- "compiler_builtins",
- "core",
-]
-
 [[package]]
 name = "rustc_ast_lowering"
 version = "0.0.0"
@@ -3674,17 +3663,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "rustc_lsan"
-version = "0.0.0"
-dependencies = [
- "alloc",
- "build_helper",
- "cmake",
- "compiler_builtins",
- "core",
-]
-
 [[package]]
 name = "rustc_macros"
 version = "0.1.0"
@@ -3746,17 +3724,6 @@ dependencies = [
  "syntax",
 ]
 
-[[package]]
-name = "rustc_msan"
-version = "0.0.0"
-dependencies = [
- "alloc",
- "build_helper",
- "cmake",
- "compiler_builtins",
- "core",
-]
-
 [[package]]
 name = "rustc_parse"
 version = "0.0.0"
@@ -3929,17 +3896,6 @@ dependencies = [
  "syntax",
 ]
 
-[[package]]
-name = "rustc_tsan"
-version = "0.0.0"
-dependencies = [
- "alloc",
- "build_helper",
- "cmake",
- "compiler_builtins",
- "core",
-]
-
 [[package]]
 name = "rustc_typeck"
 version = "0.0.0"
@@ -4301,10 +4257,6 @@ dependencies = [
  "panic_unwind",
  "profiler_builtins",
  "rand 0.7.0",
- "rustc_asan",
- "rustc_lsan",
- "rustc_msan",
- "rustc_tsan",
  "unwind",
  "wasi 0.9.0+wasi-snapshot-preview1",
 ]

src/bootstrap/builder.rs

@@ -343,6 +343,7 @@ impl<'a> Builder<'a> {
                 tool::Rustdoc,
                 tool::Clippy,
                 native::Llvm,
+                native::Sanitizers,
                 tool::Rustfmt,
                 tool::Miri,
                 native::Lld

src/bootstrap/check.rs

@@ -45,7 +45,7 @@ impl Step for Std {
         let compiler = builder.compiler(0, builder.config.build);
 
         let mut cargo = builder.cargo(compiler, Mode::Std, target, cargo_subcommand(builder.kind));
-        std_cargo(builder, &compiler, target, &mut cargo);
+        std_cargo(builder, target, &mut cargo);
 
         builder.info(&format!("Checking std artifacts ({} -> {})", &compiler.host, target));
         run_cargo(

src/bootstrap/compile.rs

@@ -87,7 +87,7 @@ impl Step for Std {
         target_deps.extend(copy_third_party_objects(builder, &compiler, target).into_iter());
 
         let mut cargo = builder.cargo(compiler, Mode::Std, target, "build");
-        std_cargo(builder, &compiler, target, &mut cargo);
+        std_cargo(builder, target, &mut cargo);
 
         builder.info(&format!(
             "Building stage{} std artifacts ({} -> {})",
@@ -153,17 +153,18 @@ fn copy_third_party_objects(
         copy_and_stamp(Path::new(&src), "libunwind.a");
     }
 
+    if builder.config.sanitizers && compiler.stage != 0 {
+        // The sanitizers are only copied in stage1 or above,
+        // to avoid creating dependency on LLVM.
+        target_deps.extend(copy_sanitizers(builder, &compiler, target));
+    }
+
     target_deps
 }
 
 /// Configure cargo to compile the standard library, adding appropriate env vars
 /// and such.
-pub fn std_cargo(
-    builder: &Builder<'_>,
-    compiler: &Compiler,
-    target: Interned<String>,
-    cargo: &mut Cargo,
-) {
+pub fn std_cargo(builder: &Builder<'_>, target: Interned<String>, cargo: &mut Cargo) {
     if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") {
         cargo.env("MACOSX_DEPLOYMENT_TARGET", target);
     }
@@ -206,19 +207,6 @@ pub fn std_cargo(
         let mut features = builder.std_features();
         features.push_str(&compiler_builtins_c_feature);
 
-        if compiler.stage != 0 && builder.config.sanitizers {
-            // This variable is used by the sanitizer runtime crates, e.g.
-            // rustc_lsan, to build the sanitizer runtime from C code
-            // When this variable is missing, those crates won't compile the C code,
-            // so we don't set this variable during stage0 where llvm-config is
-            // missing
-            // We also only build the runtimes when --enable-sanitizers (or its
-            // config.toml equivalent) is used
-            let llvm_config = builder.ensure(native::Llvm { target: builder.config.build });
-            cargo.env("LLVM_CONFIG", llvm_config);
-            cargo.env("RUSTC_BUILD_SANITIZERS", "1");
-        }
-
         cargo
             .arg("--features")
             .arg(features)
@@ -276,31 +264,43 @@ impl Step for StdLink {
         let libdir = builder.sysroot_libdir(target_compiler, target);
         let hostdir = builder.sysroot_libdir(target_compiler, compiler.host);
         add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));
-
-        if builder.config.sanitizers && compiler.stage != 0 && target == "x86_64-apple-darwin" {
-            // The sanitizers are only built in stage1 or above, so the dylibs will
-            // be missing in stage0 and causes panic. See the `std()` function above
-            // for reason why the sanitizers are not built in stage0.
-            copy_apple_sanitizer_dylibs(builder, &builder.native_dir(target), "osx", &libdir);
-        }
     }
 }
 
-fn copy_apple_sanitizer_dylibs(
+/// Copies sanitizer runtime libraries into target libdir.
+fn copy_sanitizers(
     builder: &Builder<'_>,
-    native_dir: &Path,
-    platform: &str,
-    into: &Path,
-) {
-    for &sanitizer in &["asan", "tsan"] {
-        let filename = format!("lib__rustc__clang_rt.{}_{}_dynamic.dylib", sanitizer, platform);
-        let mut src_path = native_dir.join(sanitizer);
-        src_path.push("build");
-        src_path.push("lib");
-        src_path.push("darwin");
-        src_path.push(&filename);
-        builder.copy(&src_path, &into.join(filename));
+    compiler: &Compiler,
+    target: Interned<String>,
+) -> Vec<PathBuf> {
+    let runtimes: Vec<native::SanitizerRuntime> = builder.ensure(native::Sanitizers { target });
+
+    if builder.config.dry_run {
+        return Vec::new();
+    }
+
+    let mut target_deps = Vec::new();
+    let libdir = builder.sysroot_libdir(*compiler, target);
+
+    for runtime in &runtimes {
+        let dst = libdir.join(&runtime.name);
+        builder.copy(&runtime.path, &dst);
+
+        if target == "x86_64-apple-darwin" {
+            // Update the library install name reflect the fact it has been renamed.
+            let status = Command::new("install_name_tool")
+                .arg("-id")
+                .arg(format!("@rpath/{}", runtime.name))
+                .arg(&dst)
+                .status()
+                .expect("failed to execute `install_name_tool`");
+            assert!(status.success());
+        }
+
+        target_deps.push(dst);
     }
+
+    target_deps
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]

src/bootstrap/dist.rs

@@ -984,10 +984,6 @@ impl Step for Src {
             "src/libcore",
             "src/libpanic_abort",
             "src/libpanic_unwind",
-            "src/librustc_asan",
-            "src/librustc_lsan",
-            "src/librustc_msan",
-            "src/librustc_tsan",
             "src/libstd",
             "src/libunwind",
             "src/libtest",

src/bootstrap/doc.rs

@@ -391,7 +391,7 @@ impl Step for Std {
 
         let run_cargo_rustdoc_for = |package: &str| {
             let mut cargo = builder.cargo(compiler, Mode::Std, target, "rustdoc");
-            compile::std_cargo(builder, &compiler, target, &mut cargo);
+            compile::std_cargo(builder, target, &mut cargo);
 
             // Keep a whitelist so we do not build internal stdlib crates, these will be
             // build by the rustc step later if enabled.

src/bootstrap/native.rs

@@ -546,3 +546,118 @@ impl Step for TestHelpers {
             .compile("rust_test_helpers");
     }
 }
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Sanitizers {
+    pub target: Interned<String>,
+}
+
+impl Step for Sanitizers {
+    type Output = Vec<SanitizerRuntime>;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/llvm-project/compiler-rt").path("src/sanitizers")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Sanitizers { target: run.target });
+    }
+
+    /// Builds sanitizer runtime libraries.
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let compiler_rt_dir = builder.src.join("src/llvm-project/compiler-rt");
+        if !compiler_rt_dir.exists() {
+            return Vec::new();
+        }
+
+        let out_dir = builder.native_dir(self.target).join("sanitizers");
+        let runtimes = supported_sanitizers(&out_dir, self.target);
+        if runtimes.is_empty() {
+            return runtimes;
+        }
+
+        let llvm_config = builder.ensure(Llvm { target: builder.config.build });
+        if builder.config.dry_run {
+            return runtimes;
+        }
+
+        let done_stamp = out_dir.join("sanitizers-finished-building");
+        if done_stamp.exists() {
+            builder.info(&format!(
+                "Assuming that sanitizers rebuild is not necessary. \
+                To force a rebuild, remove the file `{}`",
+                done_stamp.display()
+            ));
+            return runtimes;
+        }
+
+        builder.info(&format!("Building sanitizers for {}", self.target));
+        let _time = util::timeit(&builder);
+
+        let mut cfg = cmake::Config::new(&compiler_rt_dir);
+        cfg.target(&self.target);
+        cfg.host(&builder.config.build);
+        cfg.profile("Release");
+
+        cfg.define("CMAKE_C_COMPILER_TARGET", self.target);
+        cfg.define("COMPILER_RT_BUILD_BUILTINS", "OFF");
+        cfg.define("COMPILER_RT_BUILD_CRT", "OFF");
+        cfg.define("COMPILER_RT_BUILD_LIBFUZZER", "OFF");
+        cfg.define("COMPILER_RT_BUILD_PROFILE", "OFF");
+        cfg.define("COMPILER_RT_BUILD_SANITIZERS", "ON");
+        cfg.define("COMPILER_RT_BUILD_XRAY", "OFF");
+        cfg.define("COMPILER_RT_DEFAULT_TARGET_ONLY", "ON");
+        cfg.define("COMPILER_RT_USE_LIBCXX", "OFF");
+        cfg.define("LLVM_CONFIG_PATH", &llvm_config);
+
+        t!(fs::create_dir_all(&out_dir));
+        cfg.out_dir(out_dir);
+
+        for runtime in &runtimes {
+            cfg.build_target(&runtime.cmake_target);
+            cfg.build();
+        }
+
+        t!(fs::write(&done_stamp, b""));
+
+        runtimes
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct SanitizerRuntime {
+    /// CMake target used to build the runtime.
+    pub cmake_target: String,
+    /// Path to the built runtime library.
+    pub path: PathBuf,
+    /// Library filename that will be used rustc.
+    pub name: String,
+}
+
+/// Returns sanitizers available on a given target.
+fn supported_sanitizers(out_dir: &Path, target: Interned<String>) -> Vec<SanitizerRuntime> {
+    let mut result = Vec::new();
+    match &*target {
+        "x86_64-apple-darwin" => {
+            for s in &["asan", "lsan", "tsan"] {
+                result.push(SanitizerRuntime {
+                    cmake_target: format!("clang_rt.{}_osx_dynamic", s),
+                    path: out_dir
+                        .join(&format!("build/lib/darwin/libclang_rt.{}_osx_dynamic.dylib", s)),
+                    name: format!("librustc_rt.{}.dylib", s),
+                });
+            }
+        }
+        "x86_64-unknown-linux-gnu" => {
+            for s in &["asan", "lsan", "msan", "tsan"] {
+                result.push(SanitizerRuntime {
+                    cmake_target: format!("clang_rt.{}-x86_64", s),
+                    path: out_dir.join(&format!("build/lib/linux/libclang_rt.{}-x86_64.a", s)),
+                    name: format!("librustc_rt.{}.a", s),
+                });
+            }
+        }
+        _ => {}
+    }
+    result
+}

src/bootstrap/test.rs

@@ -1659,7 +1659,7 @@ impl Step for Crate {
         let mut cargo = builder.cargo(compiler, mode, target, test_kind.subcommand());
         match mode {
             Mode::Std => {
-                compile::std_cargo(builder, &compiler, target, &mut cargo);
+                compile::std_cargo(builder, target, &mut cargo);
             }
             Mode::Rustc => {
                 builder.ensure(compile::Rustc { compiler, target });