Improve Rustdoc's handling of procedural macros

Fixes rust-lang#58700
Fixes rust-lang#58696
Fixes rust-lang#49553
Fixes rust-lang#52210

This commit removes the special rustdoc handling for proc macros, as we
can now
retrieve their span and attributes just like any other item.

A new command-line option is added to rustdoc: `--crate-type`. This
takes the same options as rustc's `--crate-type` option. However, all
values other than `proc-macro` are treated the same. This allows Rustdoc
to enable 'proc macro mode' when handling a proc macro crate.

In compiletest, a new 'rustdoc-flags' option is added. This allows us to
pass in the '--proc-macro-crate' flag in the absence of Cargo.

I've opened [an additional PR to
Cargo](rust-lang/cargo#7159) to support passing
in this flag.
These two PRS can be merged in any order - the Cargo changes will not
take effect until the 'cargo' submodule is updated in this repository.

src/librustc/session/config.rs

@@ -1719,13 +1719,7 @@ pub fn rustc_short_optgroups() -> Vec<RustcOptGroup> {
                              static, framework, or dylib (the default).",
             "[KIND=]NAME",
         ),
-        opt::multi_s(
-            "",
-            "crate-type",
-            "Comma separated list of types of crates
-                                    for the compiler to emit",
-            "[bin|lib|rlib|dylib|cdylib|staticlib|proc-macro]",
-        ),
+        make_crate_type_option(),
         opt::opt_s(
             "",
             "crate-name",
@@ -2506,6 +2500,16 @@ pub fn build_session_options_and_crate_config(
     )
 }
 
+pub fn make_crate_type_option() -> RustcOptGroup {
+    opt::multi_s(
+        "",
+        "crate-type",
+        "Comma separated list of types of crates
+                                for the compiler to emit",
+        "[bin|lib|rlib|dylib|cdylib|staticlib|proc-macro]",
+    )
+}
+
 pub fn parse_crate_types_from_list(list_list: Vec<String>) -> Result<Vec<CrateType>, String> {
     let mut crate_types: Vec<CrateType> = Vec::new();
     for unparsed_crate_type in &list_list {

src/librustc_interface/passes.rs

@@ -473,27 +473,22 @@ fn configure_and_expand_inner<'a>(
         ast_validation::check_crate(sess, &krate)
     });
 
-    // If we're in rustdoc we're always compiling as an rlib, but that'll trip a
-    // bunch of checks in the `modify` function below. For now just skip this
-    // step entirely if we're rustdoc as it's not too useful anyway.
-    if !sess.opts.actually_rustdoc {
-        krate = time(sess, "maybe creating a macro crate", || {
-            let crate_types = sess.crate_types.borrow();
-            let num_crate_types = crate_types.len();
-            let is_proc_macro_crate = crate_types.contains(&config::CrateType::ProcMacro);
-            let is_test_crate = sess.opts.test;
-            syntax_ext::proc_macro_harness::inject(
-                &sess.parse_sess,
-                &mut resolver,
-                krate,
-                is_proc_macro_crate,
-                has_proc_macro_decls,
-                is_test_crate,
-                num_crate_types,
-                sess.diagnostic(),
-            )
-        });
-    }
+    krate = time(sess, "maybe creating a macro crate", || {
+        let crate_types = sess.crate_types.borrow();
+        let num_crate_types = crate_types.len();
+        let is_proc_macro_crate = crate_types.contains(&config::CrateType::ProcMacro);
+        let is_test_crate = sess.opts.test;
+        syntax_ext::proc_macro_harness::inject(
+            &sess.parse_sess,
+            &mut resolver,
+            krate,
+            is_proc_macro_crate,
+            has_proc_macro_decls,
+            is_test_crate,
+            num_crate_types,
+            sess.diagnostic(),
+        )
+    });
 
     // Done with macro expansion!
 

src/librustdoc/clean/inline.rs

@@ -20,6 +20,7 @@ use crate::clean::{
     self,
     GetDefId,
     ToSource,
+    TypeKind
 };
 
 use super::Clean;
@@ -107,15 +108,16 @@ pub fn try_inline(
             record_extern_fqn(cx, did, clean::TypeKind::Const);
             clean::ConstantItem(build_const(cx, did))
         }
-        // FIXME: proc-macros don't propagate attributes or spans across crates, so they look empty
-        Res::Def(DefKind::Macro(MacroKind::Bang), did) => {
+        Res::Def(DefKind::Macro(kind), did) => {
             let mac = build_macro(cx, did, name);
-            if let clean::MacroItem(..) = mac {
-                record_extern_fqn(cx, did, clean::TypeKind::Macro);
-                mac
-            } else {
-                return None;
-            }
+
+            let type_kind = match kind {
+                MacroKind::Bang => TypeKind::Macro,
+                MacroKind::Attr => TypeKind::Attr,
+                MacroKind::Derive => TypeKind::Derive
+            };
+            record_extern_fqn(cx, did, type_kind);
+            mac
         }
         _ => return None,
     };

src/librustdoc/config.rs

@@ -6,6 +6,7 @@ use errors;
 use getopts;
 use rustc::lint::Level;
 use rustc::session;
+use rustc::session::config::{CrateType, parse_crate_types_from_list};
 use rustc::session::config::{CodegenOptions, DebuggingOptions, ErrorOutputType, Externs};
 use rustc::session::config::{nightly_options, build_codegen_options, build_debugging_options,
                              get_cmd_lint_options, ExternEntry};
@@ -32,6 +33,8 @@ pub struct Options {
     pub input: PathBuf,
     /// The name of the crate being documented.
     pub crate_name: Option<String>,
+    /// Whether or not this is a proc-macro crate
+    pub proc_macro_crate: bool,
     /// How to format errors and warnings.
     pub error_format: ErrorOutputType,
     /// Library search paths to hand to the compiler.
@@ -111,6 +114,7 @@ impl fmt::Debug for Options {
         f.debug_struct("Options")
             .field("input", &self.input)
             .field("crate_name", &self.crate_name)
+            .field("proc_macro_crate", &self.proc_macro_crate)
             .field("error_format", &self.error_format)
             .field("libs", &self.libs)
             .field("externs", &FmtExterns(&self.externs))
@@ -431,7 +435,16 @@ impl Options {
         };
         let manual_passes = matches.opt_strs("passes");
 
+        let crate_types = match parse_crate_types_from_list(matches.opt_strs("crate-type")) {
+            Ok(types) => types,
+            Err(e) =>{
+                diag.struct_err(&format!("unknown crate type: {}", e)).emit();
+                return Err(1);
+            }
+        };
+
         let crate_name = matches.opt_str("crate-name");
+        let proc_macro_crate = crate_types.contains(&CrateType::ProcMacro);
         let playground_url = matches.opt_str("playground-url");
         let maybe_sysroot = matches.opt_str("sysroot").map(PathBuf::from);
         let display_warnings = matches.opt_present("display-warnings");
@@ -454,6 +467,7 @@ impl Options {
         Ok(Options {
             input,
             crate_name,
+            proc_macro_crate,
             error_format,
             libs,
             externs,

src/librustdoc/core.rs

@@ -228,6 +228,7 @@ pub fn run_core(options: RustdocOptions) -> (clean::Crate, RenderInfo, RenderOpt
     let RustdocOptions {
         input,
         crate_name,
+        proc_macro_crate,
         error_format,
         libs,
         externs,
@@ -293,11 +294,16 @@ pub fn run_core(options: RustdocOptions) -> (clean::Crate, RenderInfo, RenderOpt
     }).collect();
 
     let host_triple = TargetTriple::from_triple(config::host_triple());
+    let crate_types = if proc_macro_crate {
+        vec![config::CrateType::ProcMacro]
+    } else {
+        vec![config::CrateType::Rlib]
+    };
     // plays with error output here!
     let sessopts = config::Options {
         maybe_sysroot,
         search_paths: libs,
-        crate_types: vec![config::CrateType::Rlib],
+        crate_types,
         lint_opts: if !display_warnings {
             lint_opts
         } else {

src/librustdoc/lib.rs

@@ -46,7 +46,7 @@ use std::panic;
 use std::process;
 
 use rustc::session::{early_warn, early_error};
-use rustc::session::config::{ErrorOutputType, RustcOptGroup};
+use rustc::session::config::{ErrorOutputType, RustcOptGroup, make_crate_type_option};
 
 #[macro_use]
 mod externalfiles;
@@ -132,6 +132,7 @@ fn opts() -> Vec<RustcOptGroup> {
         stable("crate-name", |o| {
             o.optopt("", "crate-name", "specify the name of this crate", "NAME")
         }),
+        make_crate_type_option(),
         stable("L", |o| {
             o.optmulti("L", "library-path", "directory to add to crate search path",
                        "DIR")

src/librustdoc/test.rs

@@ -43,10 +43,16 @@ pub struct TestOptions {
 pub fn run(options: Options) -> i32 {
     let input = config::Input::File(options.input.clone());
 
+    let crate_types = if options.proc_macro_crate {
+        vec![config::CrateType::ProcMacro]
+    } else {
+        vec![config::CrateType::Dylib]
+    };
+
     let sessopts = config::Options {
         maybe_sysroot: options.maybe_sysroot.clone(),
         search_paths: options.libs.clone(),
-        crate_types: vec![config::CrateType::Dylib],
+        crate_types,
         cg: options.codegen_options.clone(),
         externs: options.externs.clone(),
         unstable_features: UnstableFeatures::from_environment(),

src/test/rustdoc-ui/failed-doctest-output.rs

@@ -3,6 +3,7 @@
 // adapted to use that, and that normalize line can go away
 
 // compile-flags:--test
+// rustc-env:RUST_BACKTRACE=0
 // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR"
 // failure-status: 101
 

src/test/rustdoc-ui/failed-doctest-output.stdout

@@ -1,13 +1,13 @@
 
 running 2 tests
-test $DIR/failed-doctest-output.rs - OtherStruct (line 20) ... FAILED
-test $DIR/failed-doctest-output.rs - SomeStruct (line 10) ... FAILED
+test $DIR/failed-doctest-output.rs - OtherStruct (line 21) ... FAILED
+test $DIR/failed-doctest-output.rs - SomeStruct (line 11) ... FAILED
 
 failures:
 
----- $DIR/failed-doctest-output.rs - OtherStruct (line 20) stdout ----
+---- $DIR/failed-doctest-output.rs - OtherStruct (line 21) stdout ----
 error[E0425]: cannot find value `no` in this scope
- --> $DIR/failed-doctest-output.rs:21:1
+ --> $DIR/failed-doctest-output.rs:22:1
   |
 3 | no
   | ^^ not found in this scope
@@ -16,7 +16,7 @@ error: aborting due to previous error
 
 For more information about this error, try `rustc --explain E0425`.
 Couldn't compile the test.
----- $DIR/failed-doctest-output.rs - SomeStruct (line 10) stdout ----
+---- $DIR/failed-doctest-output.rs - SomeStruct (line 11) stdout ----
 Test executable failed (exit code 101).
 
 stdout:
@@ -32,8 +32,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace.
 
 
 failures:
-    $DIR/failed-doctest-output.rs - OtherStruct (line 20)
-    $DIR/failed-doctest-output.rs - SomeStruct (line 10)
+    $DIR/failed-doctest-output.rs - OtherStruct (line 21)
+    $DIR/failed-doctest-output.rs - SomeStruct (line 11)
 
 test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out
 

src/test/rustdoc/inline_cross/auxiliary/proc_macro.rs

@@ -1,5 +1,6 @@
 // force-host
 // no-prefer-dynamic
+// compile-flags: --crate-type proc-macro
 
 #![crate_type="proc-macro"]
 #![crate_name="some_macros"]
@@ -25,3 +26,9 @@ pub fn some_proc_attr(_attr: TokenStream, item: TokenStream) -> TokenStream {
 pub fn some_derive(_item: TokenStream) -> TokenStream {
     TokenStream::new()
 }
+
+/// Doc comment from the original crate
+#[proc_macro]
+pub fn reexported_macro(_input: TokenStream) -> TokenStream {
+    TokenStream::new()
+}

src/test/rustdoc/inline_cross/proc_macro.rs

@@ -1,16 +1,17 @@
 // aux-build:proc_macro.rs
 // build-aux-docs
 
-// FIXME: if/when proc-macros start exporting their doc attributes across crates, we can turn on
-// cross-crate inlining for them
-
 extern crate some_macros;
 
 // @has proc_macro/index.html
-// @has - '//a/@href' '../some_macros/macro.some_proc_macro.html'
-// @has - '//a/@href' '../some_macros/attr.some_proc_attr.html'
-// @has - '//a/@href' '../some_macros/derive.SomeDerive.html'
-// @!has proc_macro/macro.some_proc_macro.html
-// @!has proc_macro/attr.some_proc_attr.html
-// @!has proc_macro/derive.SomeDerive.html
+// @has - '//a/@href' 'macro.some_proc_macro.html'
+// @has - '//a/@href' 'attr.some_proc_attr.html'
+// @has - '//a/@href' 'derive.SomeDerive.html'
+// @has proc_macro/macro.some_proc_macro.html
+// @has proc_macro/attr.some_proc_attr.html
+// @has proc_macro/derive.SomeDerive.html
 pub use some_macros::{some_proc_macro, some_proc_attr, SomeDerive};
+
+// @has proc_macro/macro.reexported_macro.html
+// @has - 'Doc comment from the original crate'
+pub use some_macros::reexported_macro;

src/test/rustdoc/proc-macro.rs

@@ -1,5 +1,6 @@
 // force-host
 // no-prefer-dynamic
+// compile-flags: --crate-type proc-macro --document-private-items
 
 #![crate_type="proc-macro"]
 #![crate_name="some_macros"]
@@ -58,7 +59,7 @@ pub fn some_derive(_item: TokenStream) -> TokenStream {
 }
 
 // @has some_macros/foo/index.html
-pub mod foo {
+mod foo {
     // @has - '//code' 'pub use some_proc_macro;'
     // @has - '//a/@href' '../../some_macros/macro.some_proc_macro.html'
     pub use some_proc_macro;

src/test/rustdoc/rustc-macro-crate.rs

@@ -1,5 +1,6 @@
 // force-host
 // no-prefer-dynamic
+// compile-flags: --crate-type proc-macro
 
 #![crate_type = "proc-macro"]