From 016eaf88f5bca6c9253eedec713c6ddcd6da2906 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 20 Oct 2018 19:15:06 -0700 Subject: [PATCH] Use `jemalloc-sys` on Linux and OSX compilers This commit adds opt-in support to the compiler to link to `jemalloc` in the compiler. When activated the compiler will depend on `jemalloc-sys`, instruct jemalloc to unprefix its symbols, and then link to it. The feature is activated by default on Linux/OSX compilers for x86_64/i686 platforms, and it's not enabled anywhere else for now. We may be able to opt-in other platforms in the future! Also note that the opt-in only happens on CI, it's otherwise unconditionally turned off by default. Closes #36963 --- .travis.yml | 10 +++++----- config.toml.example | 4 ++++ src/Cargo.lock | 18 ++++++++++++++++++ src/bootstrap/config.rs | 5 ++++- src/bootstrap/lib.rs | 6 +++++- src/ci/docker/dist-i686-linux/Dockerfile | 3 ++- src/ci/docker/dist-x86_64-linux/Dockerfile | 3 ++- src/librustc_driver/Cargo.toml | 5 +++++ src/librustc_driver/lib.rs | 8 ++++++++ .../spec/arm_linux_androideabi.rs | 2 +- src/rustc/Cargo.toml | 3 +++ src/tools/tidy/src/cargo.rs | 2 +- 12 files changed, 58 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3d8918095accb..b208e760d95c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ matrix: - env: > RUST_CHECK_TARGET=dist - RUST_CONFIGURE_ARGS="--enable-extended --enable-profiler --enable-lldb" + RUST_CONFIGURE_ARGS="--enable-extended --enable-profiler --enable-lldb --set rust.jemalloc" SRC=. DEPLOY_ALT=1 RUSTC_RETRY_LINKER_ON_SEGFAULT=1 @@ -53,7 +53,7 @@ matrix: # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. - env: > RUST_CHECK_TARGET=check - RUST_CONFIGURE_ARGS="--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler" + RUST_CONFIGURE_ARGS="--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc" SRC=. RUSTC_RETRY_LINKER_ON_SEGFAULT=1 MACOSX_DEPLOYMENT_TARGET=10.8 @@ -67,7 +67,7 @@ matrix: - env: > RUST_CHECK_TARGET=check - RUST_CONFIGURE_ARGS=--build=i686-apple-darwin + RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --set rust.jemalloc" SRC=. RUSTC_RETRY_LINKER_ON_SEGFAULT=1 MACOSX_DEPLOYMENT_TARGET=10.8 @@ -87,7 +87,7 @@ matrix: # OSX 10.7 and `xcode7` is the latest Xcode able to compile LLVM for 10.7. - env: > RUST_CHECK_TARGET=dist - RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb" + RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc" SRC=. DEPLOY=1 RUSTC_RETRY_LINKER_ON_SEGFAULT=1 @@ -102,7 +102,7 @@ matrix: - env: > RUST_CHECK_TARGET=dist - RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb" + RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc" SRC=. DEPLOY=1 RUSTC_RETRY_LINKER_ON_SEGFAULT=1 diff --git a/config.toml.example b/config.toml.example index 2620fec512541..8b11014edae79 100644 --- a/config.toml.example +++ b/config.toml.example @@ -396,6 +396,10 @@ # generally only set for releases #remap-debuginfo = false +# Link the compiler against `jemalloc`, where on Linux and OSX it should +# override the default allocator for rustc and LLVM. +#jemalloc = false + # ============================================================================= # Options for specific targets # diff --git a/src/Cargo.lock b/src/Cargo.lock index db111b5d2b524..dce5d4eb5129d 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -785,6 +785,11 @@ dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fs_extra" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "fst" version = "0.3.0" @@ -996,6 +1001,16 @@ name = "itoa" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "jemalloc-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "jobserver" version = "0.1.11" @@ -2181,6 +2196,7 @@ dependencies = [ "arena 0.0.0", "env_logger 0.5.12 (registry+https://github.com/rust-lang/crates.io-index)", "graphviz 0.0.0", + "jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc 0.0.0", "rustc-rayon 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3237,6 +3253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" "checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" "checksum fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +"checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" "checksum fst 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d94485a00b1827b861dd9d1a2cc9764f9044d4c535514c0760a5a2012ef3399f" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" @@ -3259,6 +3276,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum is-match 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7e5b386aef33a1c677be65237cb9d32c3f3ef56bd035949710c4bb13083eb053" "checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" +"checksum jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "bfc62c8e50e381768ce8ee0428ee53741929f7ebd73e4d83f669bcf7693e00ae" "checksum jobserver 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "60af5f849e1981434e4a31d3d782c4774ae9b434ce55b101a96ecfd09147e8be" "checksum json 0.11.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9ad0485404155f45cce53a40d4b2d6ac356418300daed05273d9e26f91c390be" "checksum jsonrpc-core 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ddf83704f4e79979a424d1082dd2c1e52683058056c9280efa19ac5f6bc9033c" diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 791d2490d97ef..0f249eee00022 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -116,6 +116,7 @@ pub struct Config { pub hosts: Vec>, pub targets: Vec>, pub local_rebuild: bool, + pub jemalloc: bool, // dist misc pub dist_sign_folder: Option, @@ -260,7 +261,7 @@ struct Llvm { link_jobs: Option, link_shared: Option, version_suffix: Option, - clang_cl: Option + clang_cl: Option, } #[derive(Deserialize, Default, Clone)] @@ -324,6 +325,7 @@ struct Rust { backtrace_on_ice: Option, verify_llvm_ir: Option, remap_debuginfo: Option, + jemalloc: Option, } /// TOML representation of how each build target is configured. @@ -537,6 +539,7 @@ impl Config { set(&mut config.rust_debuginfo_tests, rust.debuginfo_tests); set(&mut config.codegen_tests, rust.codegen_tests); set(&mut config.rust_rpath, rust.rpath); + set(&mut config.jemalloc, rust.jemalloc); set(&mut config.backtrace, rust.backtrace); set(&mut config.channel, rust.channel.clone()); set(&mut config.rust_dist_src, rust.dist_src); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index bd768d5e1a270..8159fb182a18d 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -530,7 +530,11 @@ impl Build { /// Get the space-separated set of activated features for the compiler. fn rustc_features(&self) -> String { - String::new() + let mut features = String::new(); + if self.config.jemalloc { + features.push_str("jemalloc"); + } + features } /// Component directory that Cargo will produce output into (e.g. diff --git a/src/ci/docker/dist-i686-linux/Dockerfile b/src/ci/docker/dist-i686-linux/Dockerfile index d99e409e42671..8df49f364a372 100644 --- a/src/ci/docker/dist-i686-linux/Dockerfile +++ b/src/ci/docker/dist-i686-linux/Dockerfile @@ -98,7 +98,8 @@ ENV RUST_CONFIGURE_ARGS \ --enable-sanitizers \ --enable-profiler \ --set target.i686-unknown-linux-gnu.linker=clang \ - --build=i686-unknown-linux-gnu + --build=i686-unknown-linux-gnu \ + --set rust.jemalloc ENV SCRIPT python2.7 ../x.py dist --build $HOSTS --host $HOSTS --target $HOSTS ENV CARGO_TARGET_I686_UNKNOWN_LINUX_GNU_LINKER=clang diff --git a/src/ci/docker/dist-x86_64-linux/Dockerfile b/src/ci/docker/dist-x86_64-linux/Dockerfile index 8696f72e0e388..0a2dae72f7382 100644 --- a/src/ci/docker/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/dist-x86_64-linux/Dockerfile @@ -101,7 +101,8 @@ ENV RUST_CONFIGURE_ARGS \ --set target.x86_64-unknown-linux-gnu.linker=clang \ --set target.x86_64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \ --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \ - --set llvm.thin-lto=true + --set llvm.thin-lto=true \ + --set rust.jemalloc ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang diff --git a/src/librustc_driver/Cargo.toml b/src/librustc_driver/Cargo.toml index 470c8b03d0bca..1e32f5ef6f0b6 100644 --- a/src/librustc_driver/Cargo.toml +++ b/src/librustc_driver/Cargo.toml @@ -38,3 +38,8 @@ syntax = { path = "../libsyntax" } smallvec = { version = "0.6.5", features = ["union"] } syntax_ext = { path = "../libsyntax_ext" } syntax_pos = { path = "../libsyntax_pos" } + +[dependencies.jemalloc-sys] +version = '0.1.8' +optional = true +features = ['unprefixed_malloc_on_supported_platforms'] diff --git a/src/librustc_driver/lib.rs b/src/librustc_driver/lib.rs index 276b7290c2ef0..e8fdaddaeb89c 100644 --- a/src/librustc_driver/lib.rs +++ b/src/librustc_driver/lib.rs @@ -64,6 +64,14 @@ extern crate syntax; extern crate syntax_ext; extern crate syntax_pos; +// Note that the linkage here should be all that we need, on Linux we're not +// prefixing the symbols here so this should naturally override our default +// allocator. On OSX it should override via the zone allocator. We shouldn't +// enable this by default on other platforms, so other platforms aren't handled +// here yet. +#[cfg(feature = "jemalloc-sys")] +extern crate jemalloc_sys; + use driver::CompileController; use pretty::{PpMode, UserIdentifiedItem}; diff --git a/src/librustc_target/spec/arm_linux_androideabi.rs b/src/librustc_target/spec/arm_linux_androideabi.rs index c5e3385a91ca1..055bca6c34848 100644 --- a/src/librustc_target/spec/arm_linux_androideabi.rs +++ b/src/librustc_target/spec/arm_linux_androideabi.rs @@ -14,7 +14,7 @@ pub fn target() -> TargetResult { let mut base = super::android_base::opts(); // https://developer.android.com/ndk/guides/abis.html#armeabi base.features = "+strict-align,+v5te".to_string(); - base.max_atomic_width = Some(64); + base.max_atomic_width = Some(32); Ok(Target { llvm_target: "arm-linux-androideabi".to_string(), diff --git a/src/rustc/Cargo.toml b/src/rustc/Cargo.toml index 7a31afed8c322..ec822fddef3eb 100644 --- a/src/rustc/Cargo.toml +++ b/src/rustc/Cargo.toml @@ -10,3 +10,6 @@ path = "rustc.rs" [dependencies] rustc_target = { path = "../librustc_target" } rustc_driver = { path = "../librustc_driver" } + +[features] +jemalloc = ['rustc_driver/jemalloc-sys'] diff --git a/src/tools/tidy/src/cargo.rs b/src/tools/tidy/src/cargo.rs index da0cd8b665207..466d2fa0d2bd6 100644 --- a/src/tools/tidy/src/cargo.rs +++ b/src/tools/tidy/src/cargo.rs @@ -84,7 +84,7 @@ fn verify(tomlfile: &Path, libfile: &Path, bad: &mut bool) { // This is intentional, this dependency just makes the crate available // for others later on. Cover cases - let whitelisted = whitelisted || krate.starts_with("panic"); + let whitelisted = krate.starts_with("panic"); if toml.contains("name = \"std\"") && whitelisted { continue }