From 61e89446ef6e115630faa75c985c599d739f7586 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 20 Oct 2018 19:04:42 -0700 Subject: [PATCH 1/3] Remove all jemalloc-related content This commit removes all jemalloc related submodules, configuration, etc, from the bootstrap, from the standard library, and from the compiler. This will be followed up with a change to use jemalloc specifically as part of rustc on blessed platforms. --- .gitmodules | 5 +- config.toml.example | 10 -- src/bootstrap/bootstrap.py | 5 - src/bootstrap/compile.rs | 16 +- src/bootstrap/config.rs | 14 -- src/bootstrap/configure.py | 5 - src/bootstrap/dist.rs | 3 - src/bootstrap/lib.rs | 14 +- src/bootstrap/sanity.rs | 6 - src/bootstrap/test.rs | 5 +- src/jemalloc | 1 - src/liballoc/tests/heap.rs | 3 - src/liballoc_jemalloc/Cargo.toml | 24 --- src/liballoc_jemalloc/build.rs | 151 ------------------- src/liballoc_jemalloc/lib.rs | 127 ---------------- src/liballoc_jemalloc/pthread_atfork_dummy.c | 16 -- src/librustc_target/Cargo.toml | 3 - src/librustc_target/spec/apple_base.rs | 1 - src/librustc_target/spec/apple_ios_base.rs | 4 - src/librustc_target/spec/cloudabi_base.rs | 1 - src/librustc_target/spec/dragonfly_base.rs | 1 - src/librustc_target/spec/freebsd_base.rs | 1 - src/librustc_target/spec/linux_base.rs | 1 - src/librustc_target/spec/mod.rs | 8 - src/librustc_target/spec/solaris_base.rs | 1 - src/libstd/Cargo.toml | 4 - src/libstd/alloc.rs | 9 +- src/libstd/lib.rs | 9 +- src/rustc/Cargo.toml | 3 - src/tools/tidy/src/cargo.rs | 1 - src/tools/tidy/src/lib.rs | 1 - src/tools/tidy/src/pal.rs | 2 - 32 files changed, 9 insertions(+), 446 deletions(-) delete mode 160000 src/jemalloc delete mode 100644 src/liballoc_jemalloc/Cargo.toml delete mode 100644 src/liballoc_jemalloc/build.rs delete mode 100644 src/liballoc_jemalloc/lib.rs delete mode 100644 src/liballoc_jemalloc/pthread_atfork_dummy.c diff --git a/.gitmodules b/.gitmodules index 038237aa179a9..bf9bdd9a5b4b0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,9 +2,6 @@ path = src/llvm url = https://github.com/rust-lang/llvm.git branch = master -[submodule "src/jemalloc"] - path = src/jemalloc - url = https://github.com/rust-lang/jemalloc.git [submodule "src/rust-installer"] path = src/tools/rust-installer url = https://github.com/rust-lang/rust-installer.git @@ -64,4 +61,4 @@ path = src/tools/clang url = https://github.com/rust-lang-nursery/clang.git branch = rust-release-80-v1 - \ No newline at end of file + diff --git a/config.toml.example b/config.toml.example index 093b8f9e526ec..2620fec512541 100644 --- a/config.toml.example +++ b/config.toml.example @@ -300,12 +300,6 @@ # Adding debuginfo makes them several times larger. #debuginfo-tools = false -# Whether or not jemalloc is built and enabled -#use-jemalloc = true - -# Whether or not jemalloc is built with its debug option set -#debug-jemalloc = false - # Whether or not `panic!`s generate backtraces (RUST_BACKTRACE) #backtrace = true @@ -441,10 +435,6 @@ # not, you can specify an explicit file name for it. #llvm-filecheck = "/path/to/FileCheck" -# Path to the custom jemalloc static library to link into the standard library -# by default. This is only used if jemalloc is still enabled above -#jemalloc = "/path/to/jemalloc/libjemalloc_pic.a" - # If this target is for Android, this option will be required to specify where # the NDK for the target lives. This is used to find the C compiler to link and # build native code. diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index b5dc0090c8b9c..fdd8784453bdb 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -715,11 +715,6 @@ def update_submodules(self): backends = self.get_toml('codegen-backends') if backends is None or not 'emscripten' in backends: continue - if module.endswith("jemalloc"): - if self.get_toml('use-jemalloc') == 'false': - continue - if self.get_toml('jemalloc'): - continue if module.endswith("lld"): config = self.get_toml('lld') if config is None or config == 'false': diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index d009d9645c35a..885ad07e08736 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -158,16 +158,7 @@ pub fn std_cargo(builder: &Builder, .arg("--manifest-path") .arg(builder.src.join("src/rustc/compiler_builtins_shim/Cargo.toml")); } else { - let mut features = builder.std_features(); - - // When doing a local rebuild we tell cargo that we're stage1 rather than - // stage0. This works fine if the local rust and being-built rust have the - // same view of what the default allocator is, but fails otherwise. Since - // we don't have a way to express an allocator preference yet, work - // around the issue in the case of a local rebuild with jemalloc disabled. - if compiler.stage == 0 && builder.local_rebuild && !builder.config.use_jemalloc { - features.push_str(" force_alloc_system"); - } + let features = builder.std_features(); if compiler.stage != 0 && builder.config.sanitizers { // This variable is used by the sanitizer runtime crates, e.g. @@ -188,11 +179,6 @@ pub fn std_cargo(builder: &Builder, .arg("--manifest-path") .arg(builder.src.join("src/libstd/Cargo.toml")); - if let Some(target) = builder.config.target_config.get(&target) { - if let Some(ref jemalloc) = target.jemalloc { - cargo.env("JEMALLOC_OVERRIDE", jemalloc); - } - } if target.contains("musl") { if let Some(p) = builder.musl_root(target) { cargo.env("MUSL_ROOT", p); diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 3eb6e8d84e877..791d2490d97ef 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -123,8 +123,6 @@ pub struct Config { pub dist_gpg_password_file: Option, // libstd features - pub debug_jemalloc: bool, - pub use_jemalloc: bool, pub backtrace: bool, // support for RUST_BACKTRACE pub wasm_syscall: bool, @@ -166,7 +164,6 @@ pub struct Target { pub llvm_config: Option, /// Some(path to FileCheck) if one was specified. pub llvm_filecheck: Option, - pub jemalloc: Option, pub cc: Option, pub cxx: Option, pub ar: Option, @@ -302,8 +299,6 @@ struct Rust { debuginfo_only_std: Option, debuginfo_tools: Option, experimental_parallel_queries: Option, - debug_jemalloc: Option, - use_jemalloc: Option, backtrace: Option, default_linker: Option, channel: Option, @@ -337,7 +332,6 @@ struct Rust { struct TomlTarget { llvm_config: Option, llvm_filecheck: Option, - jemalloc: Option, cc: Option, cxx: Option, ar: Option, @@ -363,7 +357,6 @@ impl Config { config.llvm_enabled = true; config.llvm_optimize = true; config.llvm_version_check = true; - config.use_jemalloc = true; config.backtrace = true; config.rust_optimize = true; config.rust_optimize_tests = true; @@ -499,7 +492,6 @@ impl Config { let mut debuginfo_only_std = None; let mut debuginfo_tools = None; let mut debug = None; - let mut debug_jemalloc = None; let mut debuginfo = None; let mut debug_assertions = None; let mut optimize = None; @@ -541,12 +533,10 @@ impl Config { debuginfo_tools = rust.debuginfo_tools; optimize = rust.optimize; ignore_git = rust.ignore_git; - debug_jemalloc = rust.debug_jemalloc; set(&mut config.rust_optimize_tests, rust.optimize_tests); set(&mut config.rust_debuginfo_tests, rust.debuginfo_tests); set(&mut config.codegen_tests, rust.codegen_tests); set(&mut config.rust_rpath, rust.rpath); - set(&mut config.use_jemalloc, rust.use_jemalloc); set(&mut config.backtrace, rust.backtrace); set(&mut config.channel, rust.channel.clone()); set(&mut config.rust_dist_src, rust.dist_src); @@ -596,9 +586,6 @@ impl Config { if let Some(ref s) = cfg.llvm_filecheck { target.llvm_filecheck = Some(config.src.join(s)); } - if let Some(ref s) = cfg.jemalloc { - target.jemalloc = Some(config.src.join(s)); - } if let Some(ref s) = cfg.android_ndk { target.ndk = Some(config.src.join(s)); } @@ -644,7 +631,6 @@ impl Config { config.rust_debuginfo_tools = debuginfo_tools.unwrap_or(false); let default = debug == Some(true); - config.debug_jemalloc = debug_jemalloc.unwrap_or(default); config.rust_debuginfo = debuginfo.unwrap_or(default); config.rust_debug_assertions = debug_assertions.unwrap_or(default); diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index d5f8d9d27d956..c69e9a1d9dcf0 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -82,7 +82,6 @@ def v(*args): o("debuginfo-lines", "rust.debuginfo-lines", "build with line number debugger metadata") o("debuginfo-only-std", "rust.debuginfo-only-std", "build only libstd with debugging information") o("debuginfo-tools", "rust.debuginfo-tools", "build extended tools with debugging information") -o("debug-jemalloc", "rust.debug-jemalloc", "build jemalloc with --enable-debug --enable-fill") v("save-toolstates", "rust.save-toolstates", "save build and test status of external tools into this file") v("prefix", "install.prefix", "set installation prefix") @@ -99,7 +98,6 @@ def v(*args): v("llvm-config", None, "set path to llvm-config") v("llvm-filecheck", None, "set path to LLVM's FileCheck utility") v("python", "build.python", "set path to python") -v("jemalloc-root", None, "set directory where libjemalloc_pic.a is located") v("android-cross-path", "target.arm-linux-androideabi.android-ndk", "Android NDK standalone path (deprecated)") v("i686-linux-android-ndk", "target.i686-linux-android.android-ndk", @@ -148,7 +146,6 @@ def v(*args): # Many of these are saved below during the "writing configuration" step # (others are conditionally saved). o("manage-submodules", "build.submodules", "let the build manage the git submodules") -o("jemalloc", "rust.use-jemalloc", "build liballoc with jemalloc") o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two") o("extended", "build.extended", "build an extended rust tool set") @@ -330,8 +327,6 @@ def set(key, value): set('target.{}.llvm-config'.format(build()), value) elif option.name == 'llvm-filecheck': set('target.{}.llvm-filecheck'.format(build()), value) - elif option.name == 'jemalloc-root': - set('target.{}.jemalloc'.format(build()), value + '/libjemalloc_pic.a') elif option.name == 'tools': set('build.tools', value.split(',')) elif option.name == 'host': diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 6d40575352418..4e8c87312e6a2 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -859,7 +859,6 @@ impl Step for Src { "src/build_helper", "src/dlmalloc", "src/liballoc", - "src/liballoc_jemalloc", "src/liballoc_system", "src/libbacktrace", "src/libcompiler_builtins", @@ -878,14 +877,12 @@ impl Step for Src { "src/rustc/dlmalloc_shim", "src/libtest", "src/libterm", - "src/jemalloc", "src/libprofiler_builtins", "src/stdsimd", "src/libproc_macro", ]; let std_src_dirs_exclude = [ "src/libcompiler_builtins/compiler-rt/test", - "src/jemalloc/test/unit", ]; copy_src_dirs(builder, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index ba601249ea895..bd768d5e1a270 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -516,12 +516,6 @@ impl Build { fn std_features(&self) -> String { let mut features = "panic-unwind".to_string(); - if self.config.debug_jemalloc { - features.push_str(" debug-jemalloc"); - } - if self.config.use_jemalloc { - features.push_str(" jemalloc"); - } if self.config.backtrace { features.push_str(" backtrace"); } @@ -536,11 +530,7 @@ impl Build { /// Get the space-separated set of activated features for the compiler. fn rustc_features(&self) -> String { - let mut features = String::new(); - if self.config.use_jemalloc { - features.push_str(" jemalloc"); - } - features + String::new() } /// Component directory that Cargo will produce output into (e.g. @@ -791,7 +781,7 @@ impl Build { // If we're compiling on macOS then we add a few unconditional flags // indicating that we want libc++ (more filled out than libstdc++) and // we want to compile for 10.7. This way we can ensure that - // LLVM/jemalloc/etc are all properly compiled. + // LLVM/etc are all properly compiled. if target.contains("apple-darwin") { base.push("-stdlib=libc++".into()); } diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 2eb11b06e4e08..15d3bccba09cc 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -152,12 +152,6 @@ pub fn check(build: &mut Build) { if !build.config.dry_run { cmd_finder.must_have(build.cxx(*host).unwrap()); } - - // The msvc hosts don't use jemalloc, turn it off globally to - // avoid packaging the dummy liballoc_jemalloc on that platform. - if host.contains("msvc") { - build.config.use_jemalloc = false; - } } // Externally configured LLVM requires FileCheck to exist diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index e411e0d17342e..e55773011df8e 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1504,8 +1504,7 @@ impl Step for CrateNotDefault { type Output = (); fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/liballoc_jemalloc") - .path("src/librustc_asan") + run.path("src/librustc_asan") .path("src/librustc_lsan") .path("src/librustc_msan") .path("src/librustc_tsan") @@ -1522,7 +1521,6 @@ impl Step for CrateNotDefault { target: run.target, test_kind, krate: match run.path { - _ if run.path.ends_with("src/liballoc_jemalloc") => "alloc_jemalloc", _ if run.path.ends_with("src/librustc_asan") => "rustc_asan", _ if run.path.ends_with("src/librustc_lsan") => "rustc_lsan", _ if run.path.ends_with("src/librustc_msan") => "rustc_msan", @@ -1561,7 +1559,6 @@ impl Step for Crate { run = run.krate("test"); for krate in run.builder.in_tree_crates("std") { if krate.is_local(&run.builder) - && !krate.name.contains("jemalloc") && !(krate.name.starts_with("rustc_") && krate.name.ends_with("san")) && krate.name != "dlmalloc" { diff --git a/src/jemalloc b/src/jemalloc deleted file mode 160000 index 1f5a28755e301..0000000000000 --- a/src/jemalloc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1f5a28755e301ac581e2048011e4e0ff3da482ef diff --git a/src/liballoc/tests/heap.rs b/src/liballoc/tests/heap.rs index 6fa88ce969a0e..b6be38107da7f 100644 --- a/src/liballoc/tests/heap.rs +++ b/src/liballoc/tests/heap.rs @@ -12,9 +12,6 @@ use alloc_system::System; use std::alloc::{Global, Alloc, Layout}; /// https://github.com/rust-lang/rust/issues/45955 -/// -/// Note that `#[global_allocator]` is not used, -/// so `liballoc_jemalloc` is linked (on some platforms). #[test] fn alloc_system_overaligned_request() { check_overalign_requests(System) diff --git a/src/liballoc_jemalloc/Cargo.toml b/src/liballoc_jemalloc/Cargo.toml deleted file mode 100644 index 7986d5dd2eb54..0000000000000 --- a/src/liballoc_jemalloc/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -name = "alloc_jemalloc" -version = "0.0.0" -build = "build.rs" -links = "jemalloc" - -[lib] -name = "alloc_jemalloc" -path = "lib.rs" -test = false -doc = false - -[dependencies] -core = { path = "../libcore" } -libc = { path = "../rustc/libc_shim" } -compiler_builtins = { path = "../rustc/compiler_builtins_shim" } - -[build-dependencies] -build_helper = { path = "../build_helper" } -cc = "1.0.1" - -[features] -debug = [] diff --git a/src/liballoc_jemalloc/build.rs b/src/liballoc_jemalloc/build.rs deleted file mode 100644 index fbda425a70bf5..0000000000000 --- a/src/liballoc_jemalloc/build.rs +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![deny(warnings)] - -extern crate build_helper; -extern crate cc; - -use std::env; -use std::path::PathBuf; -use std::process::Command; -use build_helper::{run, native_lib_boilerplate}; - -fn main() { - // FIXME: This is a hack to support building targets that don't - // support jemalloc alongside hosts that do. The jemalloc build is - // controlled by a feature of the std crate, and if that feature - // changes between targets, it invalidates the fingerprint of - // std's build script (this is a cargo bug); so we must ensure - // that the feature set used by std is the same across all - // targets, which means we have to build the alloc_jemalloc crate - // for targets like emscripten, even if we don't use it. - let target = env::var("TARGET").expect("TARGET was not set"); - let host = env::var("HOST").expect("HOST was not set"); - if target.contains("bitrig") || target.contains("emscripten") || target.contains("fuchsia") || - target.contains("msvc") || target.contains("openbsd") || target.contains("redox") || - target.contains("rumprun") || target.contains("wasm32") { - println!("cargo:rustc-cfg=dummy_jemalloc"); - return; - } - - // CloudABI ships with a copy of jemalloc that has been patched to - // work well with sandboxing. Don't attempt to build our own copy, - // as it won't build. - if target.contains("cloudabi") { - return; - } - - if target.contains("android") { - println!("cargo:rustc-link-lib=gcc"); - } else if !target.contains("windows") && !target.contains("musl") { - println!("cargo:rustc-link-lib=pthread"); - } - - if let Some(jemalloc) = env::var_os("JEMALLOC_OVERRIDE") { - let jemalloc = PathBuf::from(jemalloc); - println!("cargo:rustc-link-search=native={}", - jemalloc.parent().unwrap().display()); - let stem = jemalloc.file_stem().unwrap().to_str().unwrap(); - let name = jemalloc.file_name().unwrap().to_str().unwrap(); - let kind = if name.ends_with(".a") { - "static" - } else { - "dylib" - }; - println!("cargo:rustc-link-lib={}={}", kind, &stem[3..]); - return; - } - - let link_name = if target.contains("windows") { "jemalloc" } else { "jemalloc_pic" }; - let native = match native_lib_boilerplate("jemalloc", "jemalloc", link_name, "lib") { - Ok(native) => native, - _ => return, - }; - - let mut cmd = Command::new("sh"); - cmd.arg(native.src_dir.join("configure") - .to_str() - .unwrap() - .replace("C:\\", "/c/") - .replace("\\", "/")) - .current_dir(&native.out_dir) - // jemalloc generates Makefile deps using GCC's "-MM" flag. This means - // that GCC will run the preprocessor, and only the preprocessor, over - // jemalloc's source files. If we don't specify CPPFLAGS, then at least - // on ARM that step fails with a "Missing implementation for 32-bit - // atomic operations" error. This is because no "-march" flag will be - // passed to GCC, and then GCC won't define the - // "__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4" macro that jemalloc needs to - // select an atomic operation implementation. - .env("CPPFLAGS", env::var_os("CFLAGS").unwrap_or_default()); - - if target.contains("ios") { - cmd.arg("--disable-tls"); - } else if target.contains("android") { - // We force android to have prefixed symbols because apparently - // replacement of the libc allocator doesn't quite work. When this was - // tested (unprefixed symbols), it was found that the `realpath` - // function in libc would allocate with libc malloc (not jemalloc - // malloc), and then the standard library would free with jemalloc free, - // causing a segfault. - // - // If the test suite passes, however, without symbol prefixes then we - // should be good to go! - cmd.arg("--with-jemalloc-prefix=je_"); - cmd.arg("--disable-tls"); - } else if target.contains("dragonfly") || target.contains("musl") { - cmd.arg("--with-jemalloc-prefix=je_"); - } - - if cfg!(feature = "debug") { - // Enable jemalloc assertions. - cmd.arg("--enable-debug"); - } - - cmd.arg(format!("--host={}", build_helper::gnu_target(&target))); - cmd.arg(format!("--build={}", build_helper::gnu_target(&host))); - - // for some reason, jemalloc configure doesn't detect this value - // automatically for this target - if target == "sparc64-unknown-linux-gnu" { - cmd.arg("--with-lg-quantum=4"); - } - - run(&mut cmd); - - let mut make = Command::new(build_helper::make(&host)); - make.current_dir(&native.out_dir) - .arg("build_lib_static"); - - // These are intended for mingw32-make which we don't use - if cfg!(windows) { - make.env_remove("MAKEFLAGS").env_remove("MFLAGS"); - } - - // mingw make seems... buggy? unclear... - if !host.contains("windows") { - make.arg("-j") - .arg(env::var("NUM_JOBS").expect("NUM_JOBS was not set")); - } - - run(&mut make); - - // The pthread_atfork symbols is used by jemalloc on android but the really - // old android we're building on doesn't have them defined, so just make - // sure the symbols are available. - if target.contains("androideabi") { - println!("cargo:rerun-if-changed=pthread_atfork_dummy.c"); - cc::Build::new() - .flag("-fvisibility=hidden") - .file("pthread_atfork_dummy.c") - .compile("pthread_atfork_dummy"); - } -} diff --git a/src/liballoc_jemalloc/lib.rs b/src/liballoc_jemalloc/lib.rs deleted file mode 100644 index 0065e84a7ad1a..0000000000000 --- a/src/liballoc_jemalloc/lib.rs +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![no_std] -#![allow(unused_attributes)] -#![unstable(feature = "alloc_jemalloc", - reason = "implementation detail of std, does not provide any public API", - issue = "0")] -#![feature(core_intrinsics)] -#![feature(libc)] -#![feature(linkage)] -#![feature(nll)] -#![feature(staged_api)] -#![feature(rustc_attrs)] -#![cfg_attr(dummy_jemalloc, allow(dead_code, unused_extern_crates))] -#![cfg_attr(not(dummy_jemalloc), feature(allocator_api))] -#![rustc_alloc_kind = "exe"] - -extern crate libc; - -#[cfg(not(dummy_jemalloc))] -pub use contents::*; -#[cfg(not(dummy_jemalloc))] -mod contents { - use libc::{c_int, c_void, size_t}; - - // Note that the symbols here are prefixed by default on macOS and Windows (we - // don't explicitly request it), and on Android and DragonFly we explicitly - // request it as unprefixing cause segfaults (mismatches in allocators). - extern "C" { - #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", - target_os = "dragonfly", target_os = "windows", target_env = "musl"), - link_name = "je_mallocx")] - fn mallocx(size: size_t, flags: c_int) -> *mut c_void; - #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", - target_os = "dragonfly", target_os = "windows", target_env = "musl"), - link_name = "je_calloc")] - fn calloc(size: size_t, flags: c_int) -> *mut c_void; - #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", - target_os = "dragonfly", target_os = "windows", target_env = "musl"), - link_name = "je_rallocx")] - fn rallocx(ptr: *mut c_void, size: size_t, flags: c_int) -> *mut c_void; - #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", - target_os = "dragonfly", target_os = "windows", target_env = "musl"), - link_name = "je_sdallocx")] - fn sdallocx(ptr: *mut c_void, size: size_t, flags: c_int); - } - - const MALLOCX_ZERO: c_int = 0x40; - - // The minimum alignment guaranteed by the architecture. This value is used to - // add fast paths for low alignment values. - #[cfg(all(any(target_arch = "arm", - target_arch = "mips", - target_arch = "powerpc")))] - const MIN_ALIGN: usize = 8; - #[cfg(all(any(target_arch = "x86", - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "powerpc64", - target_arch = "mips64", - target_arch = "s390x", - target_arch = "sparc64")))] - const MIN_ALIGN: usize = 16; - - // MALLOCX_ALIGN(a) macro - fn mallocx_align(a: usize) -> c_int { - a.trailing_zeros() as c_int - } - - fn align_to_flags(align: usize, size: usize) -> c_int { - if align <= MIN_ALIGN && align <= size { - 0 - } else { - mallocx_align(align) - } - } - - // for symbol names src/librustc/middle/allocator.rs - // for signatures src/librustc_allocator/lib.rs - - // linkage directives are provided as part of the current compiler allocator - // ABI - - #[rustc_std_internal_symbol] - pub unsafe extern fn __rde_alloc(size: usize, align: usize) -> *mut u8 { - let flags = align_to_flags(align, size); - let ptr = mallocx(size as size_t, flags) as *mut u8; - ptr - } - - #[rustc_std_internal_symbol] - pub unsafe extern fn __rde_dealloc(ptr: *mut u8, - size: usize, - align: usize) { - let flags = align_to_flags(align, size); - sdallocx(ptr as *mut c_void, size, flags); - } - - #[rustc_std_internal_symbol] - pub unsafe extern fn __rde_realloc(ptr: *mut u8, - _old_size: usize, - align: usize, - new_size: usize) -> *mut u8 { - let flags = align_to_flags(align, new_size); - let ptr = rallocx(ptr as *mut c_void, new_size, flags) as *mut u8; - ptr - } - - #[rustc_std_internal_symbol] - pub unsafe extern fn __rde_alloc_zeroed(size: usize, align: usize) -> *mut u8 { - let ptr = if align <= MIN_ALIGN && align <= size { - calloc(size as size_t, 1) as *mut u8 - } else { - let flags = align_to_flags(align, size) | MALLOCX_ZERO; - mallocx(size as size_t, flags) as *mut u8 - }; - ptr - } -} diff --git a/src/liballoc_jemalloc/pthread_atfork_dummy.c b/src/liballoc_jemalloc/pthread_atfork_dummy.c deleted file mode 100644 index 4e3df0ab26c37..0000000000000 --- a/src/liballoc_jemalloc/pthread_atfork_dummy.c +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -// See comments in build.rs for why this exists -int pthread_atfork(void* prefork, - void* postfork_parent, - void* postfork_child) { - return 0; -} diff --git a/src/librustc_target/Cargo.toml b/src/librustc_target/Cargo.toml index bb686e914a048..684ea4c78978c 100644 --- a/src/librustc_target/Cargo.toml +++ b/src/librustc_target/Cargo.toml @@ -13,6 +13,3 @@ bitflags = "1.0" log = "0.4" rustc_cratesio_shim = { path = "../librustc_cratesio_shim" } serialize = { path = "../libserialize" } - -[features] -jemalloc = [] diff --git a/src/librustc_target/spec/apple_base.rs b/src/librustc_target/spec/apple_base.rs index 38b3f2528fe86..8774c15ff0121 100644 --- a/src/librustc_target/spec/apple_base.rs +++ b/src/librustc_target/spec/apple_base.rs @@ -44,7 +44,6 @@ pub fn opts() -> TargetOptions { dll_suffix: ".dylib".to_string(), archive_format: "bsd".to_string(), pre_link_args: LinkArgs::new(), - exe_allocation_crate: super::maybe_jemalloc(), has_elf_tls: version >= (10, 7), abi_return_struct_as_int: true, emit_debug_gdb_scripts: false, diff --git a/src/librustc_target/spec/apple_ios_base.rs b/src/librustc_target/spec/apple_ios_base.rs index 296eaca7c7df0..e926e4913d634 100644 --- a/src/librustc_target/spec/apple_ios_base.rs +++ b/src/librustc_target/spec/apple_ios_base.rs @@ -99,10 +99,6 @@ pub fn opts(arch: Arch) -> Result { pre_link_args, has_elf_tls: false, eliminate_frame_pointer: false, - // The following line is a workaround for jemalloc 4.5 being broken on - // ios. jemalloc 5.0 is supposed to fix this. - // see https://github.com/rust-lang/rust/issues/45262 - exe_allocation_crate: None, .. super::apple_base::opts() }) } diff --git a/src/librustc_target/spec/cloudabi_base.rs b/src/librustc_target/spec/cloudabi_base.rs index 2ffa74e737fd5..fb78cf495e22a 100644 --- a/src/librustc_target/spec/cloudabi_base.rs +++ b/src/librustc_target/spec/cloudabi_base.rs @@ -38,7 +38,6 @@ pub fn opts() -> TargetOptions { // dynamic linking. tls_model: "local-exec".to_string(), relro_level: RelroLevel::Full, - exe_allocation_crate: super::maybe_jemalloc(), .. Default::default() } } diff --git a/src/librustc_target/spec/dragonfly_base.rs b/src/librustc_target/spec/dragonfly_base.rs index 32eac8663afac..a9e317b7cb8a7 100644 --- a/src/librustc_target/spec/dragonfly_base.rs +++ b/src/librustc_target/spec/dragonfly_base.rs @@ -33,7 +33,6 @@ pub fn opts() -> TargetOptions { pre_link_args: args, position_independent_executables: true, relro_level: RelroLevel::Full, - exe_allocation_crate: super::maybe_jemalloc(), .. Default::default() } } diff --git a/src/librustc_target/spec/freebsd_base.rs b/src/librustc_target/spec/freebsd_base.rs index 04b8a6e706064..c8a2946da50a5 100644 --- a/src/librustc_target/spec/freebsd_base.rs +++ b/src/librustc_target/spec/freebsd_base.rs @@ -34,7 +34,6 @@ pub fn opts() -> TargetOptions { position_independent_executables: true, eliminate_frame_pointer: false, // FIXME 43575 relro_level: RelroLevel::Full, - exe_allocation_crate: super::maybe_jemalloc(), abi_return_struct_as_int: true, .. Default::default() } diff --git a/src/librustc_target/spec/linux_base.rs b/src/librustc_target/spec/linux_base.rs index 4a9cd9e2f3233..01f65d5736322 100644 --- a/src/librustc_target/spec/linux_base.rs +++ b/src/librustc_target/spec/linux_base.rs @@ -36,7 +36,6 @@ pub fn opts() -> TargetOptions { pre_link_args: args, position_independent_executables: true, relro_level: RelroLevel::Full, - exe_allocation_crate: super::maybe_jemalloc(), has_elf_tls: true, .. Default::default() } diff --git a/src/librustc_target/spec/mod.rs b/src/librustc_target/spec/mod.rs index d43d45f64a55c..49b4d6ec0f954 100644 --- a/src/librustc_target/spec/mod.rs +++ b/src/librustc_target/spec/mod.rs @@ -1270,14 +1270,6 @@ impl ToJson for Target { } } -fn maybe_jemalloc() -> Option { - if cfg!(feature = "jemalloc") { - Some("alloc_jemalloc".to_string()) - } else { - None - } -} - /// Either a target triple string or a path to a JSON file. #[derive(PartialEq, Clone, Debug, Hash, RustcEncodable, RustcDecodable)] pub enum TargetTriple { diff --git a/src/librustc_target/spec/solaris_base.rs b/src/librustc_target/spec/solaris_base.rs index c14cc3f5bc3be..93b889d5d399e 100644 --- a/src/librustc_target/spec/solaris_base.rs +++ b/src/librustc_target/spec/solaris_base.rs @@ -18,7 +18,6 @@ pub fn opts() -> TargetOptions { has_rpath: true, target_family: Some("unix".to_string()), is_like_solaris: true, - exe_allocation_crate: super::maybe_jemalloc(), .. Default::default() } diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml index cd1e3438fc372..0f22459b34349 100644 --- a/src/libstd/Cargo.toml +++ b/src/libstd/Cargo.toml @@ -14,7 +14,6 @@ crate-type = ["dylib", "rlib"] [dependencies] alloc = { path = "../liballoc" } -alloc_jemalloc = { path = "../liballoc_jemalloc", optional = true } alloc_system = { path = "../liballoc_system" } panic_unwind = { path = "../libpanic_unwind", optional = true } panic_abort = { path = "../libpanic_abort" } @@ -43,9 +42,6 @@ build_helper = { path = "../build_helper" } [features] backtrace = [] -debug-jemalloc = ["alloc_jemalloc/debug"] -jemalloc = ["alloc_jemalloc"] -force_alloc_system = [] panic-unwind = ["panic_unwind"] profiler = ["profiler_builtins"] diff --git a/src/libstd/alloc.rs b/src/libstd/alloc.rs index 31fc9ed3f772d..1ff342fa7a7be 100644 --- a/src/libstd/alloc.rs +++ b/src/libstd/alloc.rs @@ -13,13 +13,10 @@ //! In a given program, the standard library has one “global” memory allocator //! that is used for example by `Box` and `Vec`. //! -//! Currently the default global allocator is unspecified. -//! The compiler may link to a version of [jemalloc] on some platforms, -//! but this is not guaranteed. -//! Libraries, however, like `cdylib`s and `staticlib`s are guaranteed -//! to use the [`System`] by default. +//! Currently the default global allocator is unspecified. Libraries, however, +//! like `cdylib`s and `staticlib`s are guaranteed to use the [`System`] by +//! default. //! -//! [jemalloc]: https://github.com/jemalloc/jemalloc //! [`System`]: struct.System.html //! //! # The `#[global_allocator]` attribute diff --git a/src/libstd/lib.rs b/src/libstd/lib.rs index ab3e97d1d9241..b55d12e4c8dc9 100644 --- a/src/libstd/lib.rs +++ b/src/libstd/lib.rs @@ -313,14 +313,7 @@ #![default_lib_allocator] -// Always use alloc_system during stage0 since we don't know if the alloc_* -// crate the stage0 compiler will pick by default is enabled (e.g. -// if the user has disabled jemalloc in `./configure`). -// `force_alloc_system` is *only* intended as a workaround for local rebuilds -// with a rustc without jemalloc. -// FIXME(#44236) shouldn't need MSVC logic -#[cfg(all(not(target_env = "msvc"), - any(all(stage0, not(test)), feature = "force_alloc_system")))] +#[cfg(stage0)] #[global_allocator] static ALLOC: alloc_system::System = alloc_system::System; diff --git a/src/rustc/Cargo.toml b/src/rustc/Cargo.toml index 9ccd37a6a4592..7a31afed8c322 100644 --- a/src/rustc/Cargo.toml +++ b/src/rustc/Cargo.toml @@ -10,6 +10,3 @@ path = "rustc.rs" [dependencies] rustc_target = { path = "../librustc_target" } rustc_driver = { path = "../librustc_driver" } - -[features] -jemalloc = ["rustc_target/jemalloc"] diff --git a/src/tools/tidy/src/cargo.rs b/src/tools/tidy/src/cargo.rs index 69f61bc248dbb..da0cd8b665207 100644 --- a/src/tools/tidy/src/cargo.rs +++ b/src/tools/tidy/src/cargo.rs @@ -84,7 +84,6 @@ fn verify(tomlfile: &Path, libfile: &Path, bad: &mut bool) { // This is intentional, this dependency just makes the crate available // for others later on. Cover cases - let whitelisted = krate == "alloc_jemalloc"; let whitelisted = whitelisted || krate.starts_with("panic"); if toml.contains("name = \"std\"") && whitelisted { continue diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs index c5f5896d286c3..e235de9c5e138 100644 --- a/src/tools/tidy/src/lib.rs +++ b/src/tools/tidy/src/lib.rs @@ -57,7 +57,6 @@ pub mod libcoretest; fn filter_dirs(path: &Path) -> bool { let skip = [ "src/dlmalloc", - "src/jemalloc", "src/llvm", "src/llvm-emscripten", "src/libbacktrace", diff --git a/src/tools/tidy/src/pal.rs b/src/tools/tidy/src/pal.rs index 3d5e18e37b070..e8f197ba78afe 100644 --- a/src/tools/tidy/src/pal.rs +++ b/src/tools/tidy/src/pal.rs @@ -28,7 +28,6 @@ //! - core may not have platform-specific code //! - libcompiler_builtins may have platform-specific code //! - liballoc_system may have platform-specific code -//! - liballoc_jemalloc may have platform-specific code //! - libpanic_abort may have platform-specific code //! - libpanic_unwind may have platform-specific code //! - libunwind may have platform-specific code @@ -52,7 +51,6 @@ use std::iter::Iterator; // Paths that may contain platform-specific code const EXCEPTION_PATHS: &[&str] = &[ // std crates - "src/liballoc_jemalloc", "src/liballoc_system", "src/libcompiler_builtins", "src/liblibc", From 016eaf88f5bca6c9253eedec713c6ddcd6da2906 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 20 Oct 2018 19:15:06 -0700 Subject: [PATCH 2/3] Use `jemalloc-sys` on Linux and OSX compilers This commit adds opt-in support to the compiler to link to `jemalloc` in the compiler. When activated the compiler will depend on `jemalloc-sys`, instruct jemalloc to unprefix its symbols, and then link to it. The feature is activated by default on Linux/OSX compilers for x86_64/i686 platforms, and it's not enabled anywhere else for now. We may be able to opt-in other platforms in the future! Also note that the opt-in only happens on CI, it's otherwise unconditionally turned off by default. Closes #36963 --- .travis.yml | 10 +++++----- config.toml.example | 4 ++++ src/Cargo.lock | 18 ++++++++++++++++++ src/bootstrap/config.rs | 5 ++++- src/bootstrap/lib.rs | 6 +++++- src/ci/docker/dist-i686-linux/Dockerfile | 3 ++- src/ci/docker/dist-x86_64-linux/Dockerfile | 3 ++- src/librustc_driver/Cargo.toml | 5 +++++ src/librustc_driver/lib.rs | 8 ++++++++ .../spec/arm_linux_androideabi.rs | 2 +- src/rustc/Cargo.toml | 3 +++ src/tools/tidy/src/cargo.rs | 2 +- 12 files changed, 58 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3d8918095accb..b208e760d95c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ matrix: - env: > RUST_CHECK_TARGET=dist - RUST_CONFIGURE_ARGS="--enable-extended --enable-profiler --enable-lldb" + RUST_CONFIGURE_ARGS="--enable-extended --enable-profiler --enable-lldb --set rust.jemalloc" SRC=. DEPLOY_ALT=1 RUSTC_RETRY_LINKER_ON_SEGFAULT=1 @@ -53,7 +53,7 @@ matrix: # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. - env: > RUST_CHECK_TARGET=check - RUST_CONFIGURE_ARGS="--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler" + RUST_CONFIGURE_ARGS="--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc" SRC=. RUSTC_RETRY_LINKER_ON_SEGFAULT=1 MACOSX_DEPLOYMENT_TARGET=10.8 @@ -67,7 +67,7 @@ matrix: - env: > RUST_CHECK_TARGET=check - RUST_CONFIGURE_ARGS=--build=i686-apple-darwin + RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --set rust.jemalloc" SRC=. RUSTC_RETRY_LINKER_ON_SEGFAULT=1 MACOSX_DEPLOYMENT_TARGET=10.8 @@ -87,7 +87,7 @@ matrix: # OSX 10.7 and `xcode7` is the latest Xcode able to compile LLVM for 10.7. - env: > RUST_CHECK_TARGET=dist - RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb" + RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc" SRC=. DEPLOY=1 RUSTC_RETRY_LINKER_ON_SEGFAULT=1 @@ -102,7 +102,7 @@ matrix: - env: > RUST_CHECK_TARGET=dist - RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb" + RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc" SRC=. DEPLOY=1 RUSTC_RETRY_LINKER_ON_SEGFAULT=1 diff --git a/config.toml.example b/config.toml.example index 2620fec512541..8b11014edae79 100644 --- a/config.toml.example +++ b/config.toml.example @@ -396,6 +396,10 @@ # generally only set for releases #remap-debuginfo = false +# Link the compiler against `jemalloc`, where on Linux and OSX it should +# override the default allocator for rustc and LLVM. +#jemalloc = false + # ============================================================================= # Options for specific targets # diff --git a/src/Cargo.lock b/src/Cargo.lock index db111b5d2b524..dce5d4eb5129d 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -785,6 +785,11 @@ dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fs_extra" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "fst" version = "0.3.0" @@ -996,6 +1001,16 @@ name = "itoa" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "jemalloc-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "jobserver" version = "0.1.11" @@ -2181,6 +2196,7 @@ dependencies = [ "arena 0.0.0", "env_logger 0.5.12 (registry+https://github.com/rust-lang/crates.io-index)", "graphviz 0.0.0", + "jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc 0.0.0", "rustc-rayon 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3237,6 +3253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" "checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" "checksum fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +"checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" "checksum fst 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d94485a00b1827b861dd9d1a2cc9764f9044d4c535514c0760a5a2012ef3399f" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" @@ -3259,6 +3276,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum is-match 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7e5b386aef33a1c677be65237cb9d32c3f3ef56bd035949710c4bb13083eb053" "checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" +"checksum jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "bfc62c8e50e381768ce8ee0428ee53741929f7ebd73e4d83f669bcf7693e00ae" "checksum jobserver 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "60af5f849e1981434e4a31d3d782c4774ae9b434ce55b101a96ecfd09147e8be" "checksum json 0.11.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9ad0485404155f45cce53a40d4b2d6ac356418300daed05273d9e26f91c390be" "checksum jsonrpc-core 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ddf83704f4e79979a424d1082dd2c1e52683058056c9280efa19ac5f6bc9033c" diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 791d2490d97ef..0f249eee00022 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -116,6 +116,7 @@ pub struct Config { pub hosts: Vec>, pub targets: Vec>, pub local_rebuild: bool, + pub jemalloc: bool, // dist misc pub dist_sign_folder: Option, @@ -260,7 +261,7 @@ struct Llvm { link_jobs: Option, link_shared: Option, version_suffix: Option, - clang_cl: Option + clang_cl: Option, } #[derive(Deserialize, Default, Clone)] @@ -324,6 +325,7 @@ struct Rust { backtrace_on_ice: Option, verify_llvm_ir: Option, remap_debuginfo: Option, + jemalloc: Option, } /// TOML representation of how each build target is configured. @@ -537,6 +539,7 @@ impl Config { set(&mut config.rust_debuginfo_tests, rust.debuginfo_tests); set(&mut config.codegen_tests, rust.codegen_tests); set(&mut config.rust_rpath, rust.rpath); + set(&mut config.jemalloc, rust.jemalloc); set(&mut config.backtrace, rust.backtrace); set(&mut config.channel, rust.channel.clone()); set(&mut config.rust_dist_src, rust.dist_src); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index bd768d5e1a270..8159fb182a18d 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -530,7 +530,11 @@ impl Build { /// Get the space-separated set of activated features for the compiler. fn rustc_features(&self) -> String { - String::new() + let mut features = String::new(); + if self.config.jemalloc { + features.push_str("jemalloc"); + } + features } /// Component directory that Cargo will produce output into (e.g. diff --git a/src/ci/docker/dist-i686-linux/Dockerfile b/src/ci/docker/dist-i686-linux/Dockerfile index d99e409e42671..8df49f364a372 100644 --- a/src/ci/docker/dist-i686-linux/Dockerfile +++ b/src/ci/docker/dist-i686-linux/Dockerfile @@ -98,7 +98,8 @@ ENV RUST_CONFIGURE_ARGS \ --enable-sanitizers \ --enable-profiler \ --set target.i686-unknown-linux-gnu.linker=clang \ - --build=i686-unknown-linux-gnu + --build=i686-unknown-linux-gnu \ + --set rust.jemalloc ENV SCRIPT python2.7 ../x.py dist --build $HOSTS --host $HOSTS --target $HOSTS ENV CARGO_TARGET_I686_UNKNOWN_LINUX_GNU_LINKER=clang diff --git a/src/ci/docker/dist-x86_64-linux/Dockerfile b/src/ci/docker/dist-x86_64-linux/Dockerfile index 8696f72e0e388..0a2dae72f7382 100644 --- a/src/ci/docker/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/dist-x86_64-linux/Dockerfile @@ -101,7 +101,8 @@ ENV RUST_CONFIGURE_ARGS \ --set target.x86_64-unknown-linux-gnu.linker=clang \ --set target.x86_64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \ --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \ - --set llvm.thin-lto=true + --set llvm.thin-lto=true \ + --set rust.jemalloc ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang diff --git a/src/librustc_driver/Cargo.toml b/src/librustc_driver/Cargo.toml index 470c8b03d0bca..1e32f5ef6f0b6 100644 --- a/src/librustc_driver/Cargo.toml +++ b/src/librustc_driver/Cargo.toml @@ -38,3 +38,8 @@ syntax = { path = "../libsyntax" } smallvec = { version = "0.6.5", features = ["union"] } syntax_ext = { path = "../libsyntax_ext" } syntax_pos = { path = "../libsyntax_pos" } + +[dependencies.jemalloc-sys] +version = '0.1.8' +optional = true +features = ['unprefixed_malloc_on_supported_platforms'] diff --git a/src/librustc_driver/lib.rs b/src/librustc_driver/lib.rs index 276b7290c2ef0..e8fdaddaeb89c 100644 --- a/src/librustc_driver/lib.rs +++ b/src/librustc_driver/lib.rs @@ -64,6 +64,14 @@ extern crate syntax; extern crate syntax_ext; extern crate syntax_pos; +// Note that the linkage here should be all that we need, on Linux we're not +// prefixing the symbols here so this should naturally override our default +// allocator. On OSX it should override via the zone allocator. We shouldn't +// enable this by default on other platforms, so other platforms aren't handled +// here yet. +#[cfg(feature = "jemalloc-sys")] +extern crate jemalloc_sys; + use driver::CompileController; use pretty::{PpMode, UserIdentifiedItem}; diff --git a/src/librustc_target/spec/arm_linux_androideabi.rs b/src/librustc_target/spec/arm_linux_androideabi.rs index c5e3385a91ca1..055bca6c34848 100644 --- a/src/librustc_target/spec/arm_linux_androideabi.rs +++ b/src/librustc_target/spec/arm_linux_androideabi.rs @@ -14,7 +14,7 @@ pub fn target() -> TargetResult { let mut base = super::android_base::opts(); // https://developer.android.com/ndk/guides/abis.html#armeabi base.features = "+strict-align,+v5te".to_string(); - base.max_atomic_width = Some(64); + base.max_atomic_width = Some(32); Ok(Target { llvm_target: "arm-linux-androideabi".to_string(), diff --git a/src/rustc/Cargo.toml b/src/rustc/Cargo.toml index 7a31afed8c322..ec822fddef3eb 100644 --- a/src/rustc/Cargo.toml +++ b/src/rustc/Cargo.toml @@ -10,3 +10,6 @@ path = "rustc.rs" [dependencies] rustc_target = { path = "../librustc_target" } rustc_driver = { path = "../librustc_driver" } + +[features] +jemalloc = ['rustc_driver/jemalloc-sys'] diff --git a/src/tools/tidy/src/cargo.rs b/src/tools/tidy/src/cargo.rs index da0cd8b665207..466d2fa0d2bd6 100644 --- a/src/tools/tidy/src/cargo.rs +++ b/src/tools/tidy/src/cargo.rs @@ -84,7 +84,7 @@ fn verify(tomlfile: &Path, libfile: &Path, bad: &mut bool) { // This is intentional, this dependency just makes the crate available // for others later on. Cover cases - let whitelisted = whitelisted || krate.starts_with("panic"); + let whitelisted = krate.starts_with("panic"); if toml.contains("name = \"std\"") && whitelisted { continue } From 14c6835e03b285f7c73459ca851848fe8513b7a5 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 31 Oct 2018 11:32:27 -0700 Subject: [PATCH 3/3] rustc: Wait for all codegen threads to exit This commit updates rustc to wait for all codegen threads to exit before allowing the main thread to exit. This is a stab in the dark to fix the mysterious segfaults appearing on #55238, and hopefully we'll see whether this actually fixes things in practice... --- src/Cargo.lock | 12 ------ src/librustc_codegen_llvm/back/write.rs | 57 ++++++++++++++++++++++--- src/librustc_codegen_llvm/base.rs | 57 +++++++++++++++++++++++-- 3 files changed, 104 insertions(+), 22 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index dce5d4eb5129d..2b00cde7f19d0 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -15,17 +15,6 @@ dependencies = [ "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "alloc_jemalloc" -version = "0.0.0" -dependencies = [ - "build_helper 0.1.0", - "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "compiler_builtins 0.0.0", - "core 0.0.0", - "libc 0.0.0", -] - [[package]] name = "alloc_system" version = "0.0.0" @@ -2696,7 +2685,6 @@ name = "std" version = "0.0.0" dependencies = [ "alloc 0.0.0", - "alloc_jemalloc 0.0.0", "alloc_system 0.0.0", "build_helper 0.1.0", "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs index 81619c219757b..b5ed256cef6fd 100644 --- a/src/librustc_codegen_llvm/back/write.rs +++ b/src/librustc_codegen_llvm/back/write.rs @@ -1508,6 +1508,7 @@ enum Message { }, CodegenComplete, CodegenItem, + CodegenAborted, } struct Diagnostic { @@ -1788,6 +1789,7 @@ fn start_executing_work(tcx: TyCtxt, let mut needs_lto = Vec::new(); let mut lto_import_only_modules = Vec::new(); let mut started_lto = false; + let mut codegen_aborted = false; // This flag tracks whether all items have gone through codegens let mut codegen_done = false; @@ -1805,13 +1807,19 @@ fn start_executing_work(tcx: TyCtxt, let mut llvm_start_time = None; // Run the message loop while there's still anything that needs message - // processing: + // processing. Note that as soon as codegen is aborted we simply want to + // wait for all existing work to finish, so many of the conditions here + // only apply if codegen hasn't been aborted as they represent pending + // work to be done. while !codegen_done || - work_items.len() > 0 || running > 0 || - needs_lto.len() > 0 || - lto_import_only_modules.len() > 0 || - main_thread_worker_state != MainThreadWorkerState::Idle { + (!codegen_aborted && ( + work_items.len() > 0 || + needs_lto.len() > 0 || + lto_import_only_modules.len() > 0 || + main_thread_worker_state != MainThreadWorkerState::Idle + )) + { // While there are still CGUs to be codegened, the coordinator has // to decide how to utilize the compiler processes implicit Token: @@ -1840,6 +1848,9 @@ fn start_executing_work(tcx: TyCtxt, spawn_work(cgcx, item); } } + } else if codegen_aborted { + // don't queue up any more work if codegen was aborted, we're + // just waiting for our existing children to finish } else { // If we've finished everything related to normal codegen // then it must be the case that we've got some LTO work to do. @@ -1904,7 +1915,7 @@ fn start_executing_work(tcx: TyCtxt, // Spin up what work we can, only doing this while we've got available // parallelism slots and work left to spawn. - while work_items.len() > 0 && running < tokens.len() { + while !codegen_aborted && work_items.len() > 0 && running < tokens.len() { let (item, _) = work_items.pop().unwrap(); maybe_start_llvm_timer(cgcx.config(item.module_kind()), @@ -1969,6 +1980,7 @@ fn start_executing_work(tcx: TyCtxt, if !cgcx.opts.debugging_opts.no_parallel_llvm { helper.request_token(); } + assert!(!codegen_aborted); assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); main_thread_worker_state = MainThreadWorkerState::Idle; @@ -1976,11 +1988,26 @@ fn start_executing_work(tcx: TyCtxt, Message::CodegenComplete => { codegen_done = true; + assert!(!codegen_aborted); assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); main_thread_worker_state = MainThreadWorkerState::Idle; } + // If codegen is aborted that means translation was aborted due + // to some normal-ish compiler error. In this situation we want + // to exit as soon as possible, but we want to make sure all + // existing work has finished. Flag codegen as being done, and + // then conditions above will ensure no more work is spawned but + // we'll keep executing this loop until `running` hits 0. + Message::CodegenAborted => { + assert!(!codegen_aborted); + codegen_done = true; + codegen_aborted = true; + assert_eq!(main_thread_worker_state, + MainThreadWorkerState::Codegenning); + } + // If a thread exits successfully then we drop a token associated // with that worker and update our `running` count. We may later // re-acquire a token to continue running more work. We may also not @@ -2446,6 +2473,19 @@ impl OngoingCodegen { drop(self.coordinator_send.send(Box::new(Message::CodegenComplete))); } + /// Consume this context indicating that codegen was entirely aborted, and + /// we need to exit as quickly as possible. + /// + /// This method blocks the current thread until all worker threads have + /// finished, and all worker threads should have exited or be real close to + /// exiting at this point. + pub fn codegen_aborted(self) { + // Signal to the coordinator it should spawn no more work and start + // shutdown. + drop(self.coordinator_send.send(Box::new(Message::CodegenAborted))); + drop(self.future.join()); + } + pub fn check_for_errors(&self, sess: &Session) { self.shared_emitter_main.check(sess, false); } @@ -2464,6 +2504,11 @@ impl OngoingCodegen { } } +// impl Drop for OngoingCodegen { +// fn drop(&mut self) { +// } +// } + pub(crate) fn submit_codegened_module_to_llvm(tcx: TyCtxt, module: ModuleCodegen, cost: u64) { diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index a4c7a7123b964..a55aafe8b5731 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -76,12 +76,13 @@ use rustc_data_structures::small_c_str::SmallCStr; use rustc_data_structures::sync::Lrc; use std::any::Any; +use std::cmp; use std::ffi::CString; -use std::sync::Arc; -use std::time::{Instant, Duration}; use std::i32; -use std::cmp; +use std::ops::{Deref, DerefMut}; +use std::sync::Arc; use std::sync::mpsc; +use std::time::{Instant, Duration}; use syntax_pos::Span; use syntax_pos::symbol::InternedString; use syntax::attr; @@ -820,6 +821,7 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, metadata, rx, codegen_units.len()); + let ongoing_codegen = AbortCodegenOnDrop(Some(ongoing_codegen)); // Codegen an allocator shim, if necessary. // @@ -949,7 +951,54 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, ongoing_codegen.check_for_errors(tcx.sess); assert_and_save_dep_graph(tcx); - ongoing_codegen + ongoing_codegen.into_inner() +} + +/// A curious wrapper structure whose only purpose is to call `codegen_aborted` +/// when it's dropped abnormally. +/// +/// In the process of working on rust-lang/rust#55238 a mysterious segfault was +/// stumbled upon. The segfault was never reproduced locally, but it was +/// suspected to be releated to the fact that codegen worker threads were +/// sticking around by the time the main thread was exiting, causing issues. +/// +/// This structure is an attempt to fix that issue where the `codegen_aborted` +/// message will block until all workers have finished. This should ensure that +/// even if the main codegen thread panics we'll wait for pending work to +/// complete before returning from the main thread, hopefully avoiding +/// segfaults. +/// +/// If you see this comment in the code, then it means that this workaround +/// worked! We may yet one day track down the mysterious cause of that +/// segfault... +struct AbortCodegenOnDrop(Option); + +impl AbortCodegenOnDrop { + fn into_inner(mut self) -> OngoingCodegen { + self.0.take().unwrap() + } +} + +impl Deref for AbortCodegenOnDrop { + type Target = OngoingCodegen; + + fn deref(&self) -> &OngoingCodegen { + self.0.as_ref().unwrap() + } +} + +impl DerefMut for AbortCodegenOnDrop { + fn deref_mut(&mut self) -> &mut OngoingCodegen { + self.0.as_mut().unwrap() + } +} + +impl Drop for AbortCodegenOnDrop { + fn drop(&mut self) { + if let Some(codegen) = self.0.take() { + codegen.codegen_aborted(); + } + } } fn assert_and_save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {