rustbuild: Build documentation for proc_macro

This commit fixes rust-lang#38749 by building documentation for the `proc_macro` crate by
default for configured hosts. Unfortunately did not turn out to be a trivial
fix. Currently rustbuild generates documentation into multiple locations: one
for std, one for test, and one for rustc. The initial fix for this issue simply
actually executed `cargo doc -p proc_macro` which was otherwise completely
elided before.

Unfortunately rustbuild was the left to merge two documentation trees together.
One for the standard library and one for the rustc tree (which only had docs for
the `proc_macro` crate). Rustdoc itself knows how to merge documentation files
(specifically around search indexes, etc) but rustbuild was unaware of this, so
an initial fix ended up destroying the sidebar and the search bar from the
libstd docs.

To solve this issue the method of documentation has been tweaked slightly in
rustbuild. The build system will not use symlinks (or directory junctions on
Windows) to generate all documentation into the same location initially. This'll
rely on rustdoc's logic to weave together all the output and ensure that it ends
up all consistent.

Closes rust-lang#38749

src/bootstrap/doc.rs

@@ -19,10 +19,12 @@
 
 use std::fs::{self, File};
 use std::io::prelude::*;
+use std::io;
+use std::path::Path;
 use std::process::Command;
 
 use {Build, Compiler, Mode};
-use util::cp_r;
+use util::{cp_r, symlink_dir};
 use build_helper::up_to_date;
 
 /// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
@@ -141,7 +143,22 @@ pub fn std(build: &Build, stage: u32, target: &str) {
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
 
-    build.clear_if_dirty(&out_dir, &rustdoc);
+    // Here what we're doing is creating a *symlink* (directory junction on
+    // Windows) to the final output location. This is not done as an
+    // optimization but rather for correctness. We've got three trees of
+    // documentation, one for std, one for test, and one for rustc. It's then
+    // our job to merge them all together.
+    //
+    // Unfortunately rustbuild doesn't know nearly as well how to merge doc
+    // trees as rustdoc does itself, so instead of actually having three
+    // separate trees we just have rustdoc output to the same location across
+    // all of them.
+    //
+    // This way rustdoc generates output directly into the output, and rustdoc
+    // will also directly handle merging.
+    let my_out = build.crate_doc_out(target);
+    build.clear_if_dirty(&my_out, &rustdoc);
+    t!(symlink_dir_force(&my_out, &out_dir));
 
     let mut cargo = build.cargo(&compiler, Mode::Libstd, target, "doc");
     cargo.arg("--manifest-path")
@@ -166,7 +183,7 @@ pub fn std(build: &Build, stage: u32, target: &str) {
 
 
     build.run(&mut cargo);
-    cp_r(&out_dir, &out)
+    cp_r(&my_out, &out);
 }
 
 /// Compile all libtest documentation.
@@ -187,13 +204,16 @@ pub fn test(build: &Build, stage: u32, target: &str) {
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
 
-    build.clear_if_dirty(&out_dir, &rustdoc);
+    // See docs in std above for why we symlink
+    let my_out = build.crate_doc_out(target);
+    build.clear_if_dirty(&my_out, &rustdoc);
+    t!(symlink_dir_force(&my_out, &out_dir));
 
     let mut cargo = build.cargo(&compiler, Mode::Libtest, target, "doc");
     cargo.arg("--manifest-path")
          .arg(build.src.join("src/libtest/Cargo.toml"));
     build.run(&mut cargo);
-    cp_r(&out_dir, &out)
+    cp_r(&my_out, &out);
 }
 
 /// Generate all compiler documentation.
@@ -213,15 +233,28 @@ pub fn rustc(build: &Build, stage: u32, target: &str) {
     let out_dir = build.stage_out(&compiler, Mode::Librustc)
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
-    if !up_to_date(&rustdoc, &out_dir.join("rustc/index.html")) && out_dir.exists() {
-        t!(fs::remove_dir_all(&out_dir));
-    }
+
+    // See docs in std above for why we symlink
+    let my_out = build.crate_doc_out(target);
+    build.clear_if_dirty(&my_out, &rustdoc);
+    t!(symlink_dir_force(&my_out, &out_dir));
+
     let mut cargo = build.cargo(&compiler, Mode::Librustc, target, "doc");
     cargo.arg("--manifest-path")
          .arg(build.src.join("src/rustc/Cargo.toml"))
          .arg("--features").arg(build.rustc_features());
+
+    // Like with libstd above if compiler docs aren't enabled then we're not
+    // documenting internal dependencies, so we have a whitelist.
+    if !build.config.compiler_docs {
+        cargo.arg("--no-deps");
+        for krate in &["proc_macro"] {
+            cargo.arg("-p").arg(krate);
+        }
+    }
+
     build.run(&mut cargo);
-    cp_r(&out_dir, &out)
+    cp_r(&my_out, &out);
 }
 
 /// Generates the HTML rendered error-index by running the
@@ -240,3 +273,19 @@ pub fn error_index(build: &Build, target: &str) {
 
     build.run(&mut index);
 }
+
+fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> {
+    if let Ok(m) = fs::symlink_metadata(dst) {
+        if m.file_type().is_dir() {
+            try!(fs::remove_dir_all(dst));
+        } else {
+            // handle directory junctions on windows by falling back to
+            // `remove_dir`.
+            try!(fs::remove_file(dst).or_else(|_| {
+                fs::remove_dir(dst)
+            }));
+        }
+    }
+
+    symlink_dir(src, dst)
+}

src/bootstrap/lib.rs

@@ -707,6 +707,13 @@ impl Build {
         self.out.join(target).join("doc")
     }
 
+    /// Output directory for all crate documentation for a target (temporary)
+    ///
+    /// The artifacts here are then copied into `doc_out` above.
+    fn crate_doc_out(&self, target: &str) -> PathBuf {
+        self.out.join(target).join("crate-docs")
+    }
+
     /// Returns true if no custom `llvm-config` is set for the specified target.
     ///
     /// If no custom `llvm-config` was specified then Rust's llvm will be used.

src/bootstrap/step.rs

@@ -640,7 +640,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
         rules.doc(&krate.doc_step, path)
              .dep(|s| s.name("librustc-link"))
              .host(true)
-             .default(default && build.config.compiler_docs)
+             .default(default && build.config.docs)
              .run(move |s| doc::rustc(build, s.stage, s.target));
     }
 

src/bootstrap/util.rs

@@ -16,6 +16,7 @@
 use std::env;
 use std::ffi::OsString;
 use std::fs;
+use std::io;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::time::Instant;
@@ -183,3 +184,141 @@ impl Drop for TimeIt {
                  time.subsec_nanos() / 1_000_000);
     }
 }
+
+/// Symlinks two directories, using junctions on Windows and normal symlinks on
+/// Unix.
+pub fn symlink_dir(src: &Path, dest: &Path) -> io::Result<()> {
+    let _ = fs::remove_dir(dest);
+    return symlink_dir_inner(src, dest);
+
+    #[cfg(not(windows))]
+    fn symlink_dir_inner(src: &Path, dest: &Path) -> io::Result<()> {
+        use std::os::unix::fs;
+        fs::symlink(src, dest)
+    }
+
+    // Creating a directory junction on windows involves dealing with reparse
+    // points and the DeviceIoControl function, and this code is a skeleton of
+    // what can be found here:
+    //
+    // http://www.flexhex.com/docs/articles/hard-links.phtml
+    //
+    // Copied from std
+    #[cfg(windows)]
+    #[allow(bad_style)]
+    fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> {
+        use std::ptr;
+        use std::ffi::OsStr;
+        use std::os::windows::ffi::OsStrExt;
+
+        const MAXIMUM_REPARSE_DATA_BUFFER_SIZE: usize = 16 * 1024;
+        const GENERIC_WRITE: DWORD = 0x40000000;
+        const OPEN_EXISTING: DWORD = 3;
+        const FILE_FLAG_OPEN_REPARSE_POINT: DWORD = 0x00200000;
+        const FILE_FLAG_BACKUP_SEMANTICS: DWORD = 0x02000000;
+        const FSCTL_SET_REPARSE_POINT: DWORD = 0x900a4;
+        const IO_REPARSE_TAG_MOUNT_POINT: DWORD = 0xa0000003;
+        const FILE_SHARE_DELETE: DWORD = 0x4;
+        const FILE_SHARE_READ: DWORD = 0x1;
+        const FILE_SHARE_WRITE: DWORD = 0x2;
+
+        type BOOL = i32;
+        type DWORD = u32;
+        type HANDLE = *mut u8;
+        type LPCWSTR = *const u16;
+        type LPDWORD = *mut DWORD;
+        type LPOVERLAPPED = *mut u8;
+        type LPSECURITY_ATTRIBUTES = *mut u8;
+        type LPVOID = *mut u8;
+        type WCHAR = u16;
+        type WORD = u16;
+
+        #[repr(C)]
+        struct REPARSE_MOUNTPOINT_DATA_BUFFER {
+            ReparseTag: DWORD,
+            ReparseDataLength: DWORD,
+            Reserved: WORD,
+            ReparseTargetLength: WORD,
+            ReparseTargetMaximumLength: WORD,
+            Reserved1: WORD,
+            ReparseTarget: WCHAR,
+        }
+
+        extern "system" {
+            fn CreateFileW(lpFileName: LPCWSTR,
+                           dwDesiredAccess: DWORD,
+                           dwShareMode: DWORD,
+                           lpSecurityAttributes: LPSECURITY_ATTRIBUTES,
+                           dwCreationDisposition: DWORD,
+                           dwFlagsAndAttributes: DWORD,
+                           hTemplateFile: HANDLE)
+                           -> HANDLE;
+            fn DeviceIoControl(hDevice: HANDLE,
+                               dwIoControlCode: DWORD,
+                               lpInBuffer: LPVOID,
+                               nInBufferSize: DWORD,
+                               lpOutBuffer: LPVOID,
+                               nOutBufferSize: DWORD,
+                               lpBytesReturned: LPDWORD,
+                               lpOverlapped: LPOVERLAPPED) -> BOOL;
+        }
+
+        fn to_u16s<S: AsRef<OsStr>>(s: S) -> io::Result<Vec<u16>> {
+            Ok(s.as_ref().encode_wide().chain(Some(0)).collect())
+        }
+
+        // We're using low-level APIs to create the junction, and these are more
+        // picky about paths. For example, forward slashes cannot be used as a
+        // path separator, so we should try to canonicalize the path first.
+        let target = try!(fs::canonicalize(target));
+
+        try!(fs::create_dir(junction));
+
+        let path = try!(to_u16s(junction));
+
+        unsafe {
+            let h = CreateFileW(path.as_ptr(),
+                                GENERIC_WRITE,
+                                FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                                0 as *mut _,
+                                OPEN_EXISTING,
+                                FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS,
+                                ptr::null_mut());
+
+            let mut data = [0u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE];
+            let mut db = data.as_mut_ptr()
+                            as *mut REPARSE_MOUNTPOINT_DATA_BUFFER;
+            let buf = &mut (*db).ReparseTarget as *mut _;
+            let mut i = 0;
+            // FIXME: this conversion is very hacky
+            let v = br"\??\";
+            let v = v.iter().map(|x| *x as u16);
+            for c in v.chain(target.as_os_str().encode_wide().skip(4)) {
+                *buf.offset(i) = c;
+                i += 1;
+            }
+            *buf.offset(i) = 0;
+            i += 1;
+            (*db).ReparseTag = IO_REPARSE_TAG_MOUNT_POINT;
+            (*db).ReparseTargetMaximumLength = (i * 2) as WORD;
+            (*db).ReparseTargetLength = ((i - 1) * 2) as WORD;
+            (*db).ReparseDataLength =
+                    (*db).ReparseTargetLength as DWORD + 12;
+
+            let mut ret = 0;
+            let res = DeviceIoControl(h as *mut _,
+                                      FSCTL_SET_REPARSE_POINT,
+                                      data.as_ptr() as *mut _,
+                                      (*db).ReparseDataLength + 8,
+                                      ptr::null_mut(), 0,
+                                      &mut ret,
+                                      ptr::null_mut());
+
+            if res == 0 {
+                Err(io::Error::last_os_error())
+            } else {
+                Ok(())
+            }
+        }
+    }
+}

src/ci/docker/run.sh

@@ -54,6 +54,7 @@ exec docker \
   --env DEPLOY_ALT=$DEPLOY_ALT \
   --env LOCAL_USER_ID=`id -u` \
   --volume "$HOME/.cargo:/cargo" \
+  --privileged \
   --rm \
   rust-ci \
   /checkout/src/ci/run.sh

src/libproc_macro/lib.rs

@@ -21,7 +21,7 @@
 //! This functionality is intended to be expanded over time as more surface
 //! area for macro authors is stabilized.
 //!
-//! See [the book](../../book/procedural-macros.html) for more.
+//! See [the book](../book/procedural-macros.html) for more.
 
 #![crate_name = "proc_macro"]
 #![stable(feature = "proc_macro_lib", since = "1.15.0")]