Rollup of 10 pull requests

src/bootstrap/check.rs

@@ -550,7 +550,7 @@ fn find_tests(dir: &Path,
         let filename = e.file_name().into_string().unwrap();
         if (target.contains("windows") && filename.ends_with(".exe")) ||
            (!target.contains("windows") && !filename.contains(".")) ||
-           (target.contains("emscripten") && filename.contains(".js")){
+           (target.contains("emscripten") && filename.ends_with(".js")) {
             dst.push(e.path());
         }
     }

src/bootstrap/config.rs

@@ -59,6 +59,7 @@ pub struct Config {
     pub llvm_static_stdcpp: bool,
     pub llvm_link_shared: bool,
     pub llvm_targets: Option<String>,
+    pub llvm_link_jobs: Option<u32>,
 
     // rust codegen options
     pub rust_optimize: bool,
@@ -179,6 +180,7 @@ struct Llvm {
     version_check: Option<bool>,
     static_libstdcpp: Option<bool>,
     targets: Option<String>,
+    link_jobs: Option<u32>,
 }
 
 #[derive(RustcDecodable, Default, Clone)]
@@ -333,6 +335,7 @@ impl Config {
             set(&mut config.llvm_version_check, llvm.version_check);
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
             config.llvm_targets = llvm.targets.clone();
+            config.llvm_link_jobs = llvm.link_jobs;
         }
 
         if let Some(ref rust) = toml.rust {

src/bootstrap/config.toml.example

@@ -53,6 +53,14 @@
 # Rust team and file an issue if you need assistance in porting!
 #targets = "X86;ARM;AArch64;Mips;PowerPC;SystemZ;JSBackend;MSP430;Sparc;NVPTX"
 
+# Cap the number of parallel linker invocations when compiling LLVM.
+# This can be useful when building LLVM with debug info, which significantly
+# increases the size of binaries and consequently the memory required by
+# each linker process.
+# If absent or 0, linker invocations are treated like any other job and
+# controlled by rustbuild's -j parameter.
+#link-jobs = 0
+
 # =============================================================================
 # General build configuration options
 # =============================================================================

src/bootstrap/doc.rs

@@ -19,10 +19,12 @@
 
 use std::fs::{self, File};
 use std::io::prelude::*;
+use std::io;
+use std::path::Path;
 use std::process::Command;
 
 use {Build, Compiler, Mode};
-use util::cp_r;
+use util::{cp_r, symlink_dir};
 use build_helper::up_to_date;
 
 /// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
@@ -141,7 +143,22 @@ pub fn std(build: &Build, stage: u32, target: &str) {
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
 
-    build.clear_if_dirty(&out_dir, &rustdoc);
+    // Here what we're doing is creating a *symlink* (directory junction on
+    // Windows) to the final output location. This is not done as an
+    // optimization but rather for correctness. We've got three trees of
+    // documentation, one for std, one for test, and one for rustc. It's then
+    // our job to merge them all together.
+    //
+    // Unfortunately rustbuild doesn't know nearly as well how to merge doc
+    // trees as rustdoc does itself, so instead of actually having three
+    // separate trees we just have rustdoc output to the same location across
+    // all of them.
+    //
+    // This way rustdoc generates output directly into the output, and rustdoc
+    // will also directly handle merging.
+    let my_out = build.crate_doc_out(target);
+    build.clear_if_dirty(&my_out, &rustdoc);
+    t!(symlink_dir_force(&my_out, &out_dir));
 
     let mut cargo = build.cargo(&compiler, Mode::Libstd, target, "doc");
     cargo.arg("--manifest-path")
@@ -166,7 +183,7 @@ pub fn std(build: &Build, stage: u32, target: &str) {
 
 
     build.run(&mut cargo);
-    cp_r(&out_dir, &out)
+    cp_r(&my_out, &out);
 }
 
 /// Compile all libtest documentation.
@@ -187,13 +204,16 @@ pub fn test(build: &Build, stage: u32, target: &str) {
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
 
-    build.clear_if_dirty(&out_dir, &rustdoc);
+    // See docs in std above for why we symlink
+    let my_out = build.crate_doc_out(target);
+    build.clear_if_dirty(&my_out, &rustdoc);
+    t!(symlink_dir_force(&my_out, &out_dir));
 
     let mut cargo = build.cargo(&compiler, Mode::Libtest, target, "doc");
     cargo.arg("--manifest-path")
          .arg(build.src.join("src/libtest/Cargo.toml"));
     build.run(&mut cargo);
-    cp_r(&out_dir, &out)
+    cp_r(&my_out, &out);
 }
 
 /// Generate all compiler documentation.
@@ -213,15 +233,28 @@ pub fn rustc(build: &Build, stage: u32, target: &str) {
     let out_dir = build.stage_out(&compiler, Mode::Librustc)
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
-    if !up_to_date(&rustdoc, &out_dir.join("rustc/index.html")) && out_dir.exists() {
-        t!(fs::remove_dir_all(&out_dir));
-    }
+
+    // See docs in std above for why we symlink
+    let my_out = build.crate_doc_out(target);
+    build.clear_if_dirty(&my_out, &rustdoc);
+    t!(symlink_dir_force(&my_out, &out_dir));
+
     let mut cargo = build.cargo(&compiler, Mode::Librustc, target, "doc");
     cargo.arg("--manifest-path")
          .arg(build.src.join("src/rustc/Cargo.toml"))
          .arg("--features").arg(build.rustc_features());
+
+    // Like with libstd above if compiler docs aren't enabled then we're not
+    // documenting internal dependencies, so we have a whitelist.
+    if !build.config.compiler_docs {
+        cargo.arg("--no-deps");
+        for krate in &["proc_macro"] {
+            cargo.arg("-p").arg(krate);
+        }
+    }
+
     build.run(&mut cargo);
-    cp_r(&out_dir, &out)
+    cp_r(&my_out, &out);
 }
 
 /// Generates the HTML rendered error-index by running the
@@ -240,3 +273,15 @@ pub fn error_index(build: &Build, target: &str) {
 
     build.run(&mut index);
 }
+
+fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> {
+    if let Ok(m) = fs::symlink_metadata(dst) {
+        if m.file_type().is_dir() {
+            try!(fs::remove_dir_all(dst));
+        } else {
+            try!(fs::remove_file(dst));
+        }
+    }
+
+    symlink_dir(src, dst)
+}

src/bootstrap/lib.rs

@@ -707,6 +707,13 @@ impl Build {
         self.out.join(target).join("doc")
     }
 
+    /// Output directory for all crate documentation for a target (temporary)
+    ///
+    /// The artifacts here are then copied into `doc_out` above.
+    fn crate_doc_out(&self, target: &str) -> PathBuf {
+        self.out.join(target).join("crate-docs")
+    }
+
     /// Returns true if no custom `llvm-config` is set for the specified target.
     ///
     /// If no custom `llvm-config` was specified then Rust's llvm will be used.

src/bootstrap/native.rs

@@ -115,6 +115,12 @@ pub fn llvm(build: &Build, target: &str) {
         cfg.define("LLVM_BUILD_32_BITS", "ON");
     }
 
+    if let Some(num_linkers) = build.config.llvm_link_jobs {
+        if num_linkers > 0 {
+            cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string());
+        }
+    }
+
     // http://llvm.org/docs/HowToCrossCompileLLVM.html
     if target != build.config.build {
         // FIXME: if the llvm root for the build triple is overridden then we

src/bootstrap/step.rs

@@ -640,7 +640,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
         rules.doc(&krate.doc_step, path)
              .dep(|s| s.name("librustc-link"))
              .host(true)
-             .default(default && build.config.compiler_docs)
+             .default(default && build.config.docs)
              .run(move |s| doc::rustc(build, s.stage, s.target));
     }
 

src/bootstrap/util.rs

@@ -16,6 +16,7 @@
 use std::env;
 use std::ffi::OsString;
 use std::fs;
+use std::io;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::time::Instant;
@@ -175,3 +176,141 @@ impl Drop for TimeIt {
                  time.subsec_nanos() / 1_000_000);
     }
 }
+
+/// Symlinks two directories, using junctions on Windows and normal symlinks on
+/// Unix.
+pub fn symlink_dir(src: &Path, dest: &Path) -> io::Result<()> {
+    let _ = fs::remove_dir(dest);
+    return symlink_dir_inner(src, dest);
+
+    #[cfg(not(windows))]
+    fn symlink_dir_inner(src: &Path, dest: &Path) -> io::Result<()> {
+        use std::os::unix::fs;
+        fs::symlink(src, dest)
+    }
+
+    // Creating a directory junction on windows involves dealing with reparse
+    // points and the DeviceIoControl function, and this code is a skeleton of
+    // what can be found here:
+    //
+    // http://www.flexhex.com/docs/articles/hard-links.phtml
+    //
+    // Copied from std
+    #[cfg(windows)]
+    #[allow(bad_style)]
+    fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> {
+        use std::ptr;
+        use std::ffi::OsStr;
+        use std::os::windows::ffi::OsStrExt;
+
+        const MAXIMUM_REPARSE_DATA_BUFFER_SIZE: usize = 16 * 1024;
+        const GENERIC_WRITE: DWORD = 0x40000000;
+        const OPEN_EXISTING: DWORD = 3;
+        const FILE_FLAG_OPEN_REPARSE_POINT: DWORD = 0x00200000;
+        const FILE_FLAG_BACKUP_SEMANTICS: DWORD = 0x02000000;
+        const FSCTL_SET_REPARSE_POINT: DWORD = 0x900a4;
+        const IO_REPARSE_TAG_MOUNT_POINT: DWORD = 0xa0000003;
+        const FILE_SHARE_DELETE: DWORD = 0x4;
+        const FILE_SHARE_READ: DWORD = 0x1;
+        const FILE_SHARE_WRITE: DWORD = 0x2;
+
+        type BOOL = i32;
+        type DWORD = u32;
+        type HANDLE = *mut u8;
+        type LPCWSTR = *const u16;
+        type LPDWORD = *mut DWORD;
+        type LPOVERLAPPED = *mut u8;
+        type LPSECURITY_ATTRIBUTES = *mut u8;
+        type LPVOID = *mut u8;
+        type WCHAR = u16;
+        type WORD = u16;
+
+        #[repr(C)]
+        struct REPARSE_MOUNTPOINT_DATA_BUFFER {
+            ReparseTag: DWORD,
+            ReparseDataLength: DWORD,
+            Reserved: WORD,
+            ReparseTargetLength: WORD,
+            ReparseTargetMaximumLength: WORD,
+            Reserved1: WORD,
+            ReparseTarget: WCHAR,
+        }
+
+        extern "system" {
+            fn CreateFileW(lpFileName: LPCWSTR,
+                           dwDesiredAccess: DWORD,
+                           dwShareMode: DWORD,
+                           lpSecurityAttributes: LPSECURITY_ATTRIBUTES,
+                           dwCreationDisposition: DWORD,
+                           dwFlagsAndAttributes: DWORD,
+                           hTemplateFile: HANDLE)
+                           -> HANDLE;
+            fn DeviceIoControl(hDevice: HANDLE,
+                               dwIoControlCode: DWORD,
+                               lpInBuffer: LPVOID,
+                               nInBufferSize: DWORD,
+                               lpOutBuffer: LPVOID,
+                               nOutBufferSize: DWORD,
+                               lpBytesReturned: LPDWORD,
+                               lpOverlapped: LPOVERLAPPED) -> BOOL;
+        }
+
+        fn to_u16s<S: AsRef<OsStr>>(s: S) -> io::Result<Vec<u16>> {
+            Ok(s.as_ref().encode_wide().chain(Some(0)).collect())
+        }
+
+        // We're using low-level APIs to create the junction, and these are more
+        // picky about paths. For example, forward slashes cannot be used as a
+        // path separator, so we should try to canonicalize the path first.
+        let target = try!(fs::canonicalize(target));
+
+        try!(fs::create_dir(junction));
+
+        let path = try!(to_u16s(junction));
+
+        unsafe {
+            let h = CreateFileW(path.as_ptr(),
+                                GENERIC_WRITE,
+                                FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                                0 as *mut _,
+                                OPEN_EXISTING,
+                                FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS,
+                                ptr::null_mut());
+
+            let mut data = [0u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE];
+            let mut db = data.as_mut_ptr()
+                            as *mut REPARSE_MOUNTPOINT_DATA_BUFFER;
+            let buf = &mut (*db).ReparseTarget as *mut _;
+            let mut i = 0;
+            // FIXME: this conversion is very hacky
+            let v = br"\??\";
+            let v = v.iter().map(|x| *x as u16);
+            for c in v.chain(target.as_os_str().encode_wide().skip(4)) {
+                *buf.offset(i) = c;
+                i += 1;
+            }
+            *buf.offset(i) = 0;
+            i += 1;
+            (*db).ReparseTag = IO_REPARSE_TAG_MOUNT_POINT;
+            (*db).ReparseTargetMaximumLength = (i * 2) as WORD;
+            (*db).ReparseTargetLength = ((i - 1) * 2) as WORD;
+            (*db).ReparseDataLength =
+                    (*db).ReparseTargetLength as DWORD + 12;
+
+            let mut ret = 0;
+            let res = DeviceIoControl(h as *mut _,
+                                      FSCTL_SET_REPARSE_POINT,
+                                      data.as_ptr() as *mut _,
+                                      (*db).ReparseDataLength + 8,
+                                      ptr::null_mut(), 0,
+                                      &mut ret,
+                                      ptr::null_mut());
+
+            if res == 0 {
+                Err(io::Error::last_os_error())
+            } else {
+                Ok(())
+            }
+        }
+    }
+}

src/libproc_macro/lib.rs

@@ -21,7 +21,7 @@
 //! This functionality is intended to be expanded over time as more surface
 //! area for macro authors is stabilized.
 //!
-//! See [the book](../../book/procedural-macros.html) for more.
+//! See [the book](../book/procedural-macros.html) for more.
 
 #![crate_name = "proc_macro"]
 #![stable(feature = "proc_macro_lib", since = "1.15.0")]