make declarative macro expansion a part of query system

Cargo.lock

@@ -3809,6 +3809,7 @@ dependencies = [
  "rustc_lexer",
  "rustc_lint_defs",
  "rustc_macros",
+ "rustc_middle",
  "rustc_parse",
  "rustc_serialize",
  "rustc_session",

compiler/rustc_ast/src/token.rs

@@ -17,8 +17,9 @@ use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{edition::Edition, ErrorGuaranteed, Span, DUMMY_SP};
 use std::borrow::Cow;
 use std::fmt;
+use std::hash::Hash;
 
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, Hash)]
 pub enum CommentKind {
     Line,
     Block,

compiler/rustc_ast/src/tokenstream.rs

@@ -106,7 +106,6 @@ where
         }
     }
 }
-
 pub trait ToAttrTokenStream: sync::DynSend + sync::DynSync {
     fn to_attr_token_stream(&self) -> AttrTokenStream;
 }

compiler/rustc_expand/Cargo.toml

@@ -20,6 +20,7 @@ rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_lexer = { path = "../rustc_lexer" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }

compiler/rustc_expand/src/base.rs

@@ -16,6 +16,7 @@ use rustc_errors::{DiagCtxt, ErrorGuaranteed, PResult};
 use rustc_feature::Features;
 use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
 use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiag, RegisteredTools};
+use rustc_middle::expand::{CanRetry, TcxMacroExpander};
 use rustc_parse::{parser, MACRO_ARGUMENTS};
 use rustc_session::config::CollapseMacroDebuginfo;
 use rustc_session::{parse::ParseSess, Limit, Session};
@@ -676,6 +677,11 @@ pub enum SyntaxExtensionKind {
         Box<dyn TTMacroExpander + sync::DynSync + sync::DynSend>,
     ),
 
+    TcxLegacyBang(
+        /// An expander with signature TokenStream -> AST.
+        Lrc<dyn TcxMacroExpander + sync::DynSync + sync::DynSend>,
+    ),
+
     /// A token-based attribute macro.
     Attr(
         /// An expander with signature (TokenStream, TokenStream) -> TokenStream.
@@ -749,7 +755,9 @@ impl SyntaxExtension {
     /// Returns which kind of macro calls this syntax extension.
     pub fn macro_kind(&self) -> MacroKind {
         match self.kind {
-            SyntaxExtensionKind::Bang(..) | SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang,
+            SyntaxExtensionKind::Bang(..)
+            | SyntaxExtensionKind::LegacyBang(..)
+            | SyntaxExtensionKind::TcxLegacyBang(..) => MacroKind::Bang,
             SyntaxExtensionKind::Attr(..)
             | SyntaxExtensionKind::LegacyAttr(..)
             | SyntaxExtensionKind::NonMacroAttr => MacroKind::Attr,
@@ -1031,6 +1039,12 @@ pub trait ResolverExpand {
 
     /// Tools registered with `#![register_tool]` and used by tool attributes and lints.
     fn registered_tools(&self) -> &RegisteredTools;
+
+    fn expand_legacy_bang(
+        &self,
+        invoc_id: LocalExpnId,
+        current_expansion: LocalExpnId,
+    ) -> Result<(TokenStream, usize), CanRetry>;
 }
 
 pub trait LintStoreExpand {

compiler/rustc_expand/src/expand.rs

@@ -14,7 +14,9 @@ use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{self, try_visit, walk_list, AssocCtxt, Visitor, VisitorResult};
-use rustc_ast::{AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind};
+use rustc_ast::{
+    AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind, DUMMY_NODE_ID,
+};
 use rustc_ast::{ForeignItemKind, HasAttrs, HasNodeId};
 use rustc_ast::{Inline, ItemKind, MacStmtStyle, MetaItemKind, ModKind};
 use rustc_ast::{NestedMetaItem, NodeId, PatKind, StmtKind, TyKind};
@@ -35,11 +37,15 @@ use rustc_span::hygiene::SyntaxContext;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::{ErrorGuaranteed, FileName, LocalExpnId, Span};
 
+use crate::mbe::macro_rules::{trace_macros_note, ParserAnyMacro};
+use rustc_middle::expand::CanRetry;
+use rustc_middle::ty::TyCtxt;
 use smallvec::SmallVec;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::rc::Rc;
 use std::{iter, mem};
+use tracing::debug;
 
 macro_rules! ast_fragments {
     (
@@ -387,6 +393,18 @@ pub struct MacroExpander<'a, 'b> {
     monotonic: bool, // cf. `cx.monotonic_expander()`
 }
 
+pub fn expand_legacy_bang<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    key: (LocalExpnId, LocalExpnId),
+) -> Result<(&'tcx TokenStream, usize), CanRetry> {
+    let (invoc_id, current_expansion) = key;
+    let map = tcx.macro_map.borrow();
+    let (arg, span, expander) = map.get(&invoc_id).as_ref().unwrap();
+    expander
+        .expand(&tcx.sess, *span, arg.clone(), current_expansion)
+        .map(|(tts, i)| (tcx.arena.alloc(tts) as &TokenStream, i))
+}
+
 impl<'a, 'b> MacroExpander<'a, 'b> {
     pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self {
         MacroExpander { cx, monotonic }
@@ -672,6 +690,67 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                         Err(guar) => return ExpandResult::Ready(fragment_kind.dummy(span, guar)),
                     }
                 }
+                SyntaxExtensionKind::TcxLegacyBang(expander) => {
+                    // Macros defined in the current crate have a real node id,
+                    // whereas macros from an external crate have a dummy id.
+                    if self.cx.trace_macros() {
+                        let msg = format!(
+                            "expanding `{}! {{ {} }}`",
+                            expander.name(),
+                            pprust::tts_to_string(&mac.args.tokens)
+                        );
+                        trace_macros_note(&mut self.cx.expansions, span, msg);
+                    }
+
+                    // Macros defined in the current crate have a real node id,
+                    // whereas macros from an external crate have a dummy id.\
+                    let tok_result: Box<dyn MacResult> = match self
+                        .cx
+                        .resolver
+                        .expand_legacy_bang(invoc.expansion_data.id, self.cx.current_expansion.id)
+                    {
+                        Ok((tts, i)) => {
+                            if self.cx.trace_macros() {
+                                let msg = format!("to `{}`", pprust::tts_to_string(&tts));
+                                trace_macros_note(&mut self.cx.expansions, span, msg);
+                            }
+                            let is_local = expander.node_id() != DUMMY_NODE_ID;
+                            if is_local {
+                                self.cx.resolver.record_macro_rule_usage(expander.node_id(), i);
+                            }
+
+                            // Let the context choose how to interpret the result.
+                            // Weird, but useful for X-macros.
+                            Box::new(ParserAnyMacro::new(
+                                Parser::new(&self.cx.sess.psess, tts.clone(), None),
+                                // Pass along the original expansion site and the name of the macro,
+                                // so we can print a useful error message if the parse of the expanded
+                                // macro leaves unparsed tokens.
+                                span,
+                                expander.name(),
+                                self.cx.current_expansion.lint_node_id,
+                                self.cx.current_expansion.is_trailing_mac,
+                                expander.arm_span(i),
+                                is_local,
+                            ))
+                        }
+                        Err(CanRetry::No(guar)) => {
+                            debug!("Will not retry matching as an error was emitted already");
+                            DummyResult::any(span, guar)
+                        }
+                        Err(CanRetry::Yes) => {
+                            // Retry and emit a better error.
+                            DummyResult::any_valid(span)
+                        }
+                    };
+                    let result = if let Some(result) = fragment_kind.make_from(tok_result) {
+                        result
+                    } else {
+                        let guar = self.error_wrong_fragment_kind(fragment_kind, &mac, span);
+                        fragment_kind.dummy(span, guar)
+                    };
+                    result
+                }
                 SyntaxExtensionKind::LegacyBang(expander) => {
                     let tok_result = match expander.expand(self.cx, span, mac.args.tokens.clone()) {
                         ExpandResult::Ready(tok_result) => tok_result,

compiler/rustc_expand/src/lib.rs

@@ -26,6 +26,7 @@ mod placeholders;
 mod proc_macro_server;
 
 pub use mbe::macro_rules::compile_declarative_macro;
+use rustc_middle::query::Providers;
 pub mod base;
 pub mod config;
 pub mod expand;
@@ -34,4 +35,8 @@ pub mod module;
 #[allow(rustc::untranslatable_diagnostic)]
 pub mod proc_macro;
 
+pub fn provide(providers: &mut Providers) {
+    providers.expand_legacy_bang = expand::expand_legacy_bang;
+}
+
 rustc_fluent_macro::fluent_messages! { "../messages.ftl" }

compiler/rustc_expand/src/mbe/diagnostics.rs

@@ -18,7 +18,7 @@ use tracing::debug;
 
 use super::macro_rules::{parser_from_cx, NoopTracker};
 
-pub(super) fn failed_to_match_macro<'cx>(
+pub(crate) fn failed_to_match_macro<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
     sp: Span,
     def_span: Span,

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -28,16 +28,19 @@ use rustc_session::Session;
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::Transparency;
 use rustc_span::symbol::{kw, sym, Ident, MacroRulesNormalizedIdent};
-use rustc_span::Span;
+use rustc_span::{LocalExpnId, Span};
 use tracing::{debug, instrument, trace, trace_span};
 
+use rustc_data_structures::sync::Lrc;
 use std::borrow::Cow;
 use std::collections::hash_map::Entry;
 use std::{mem, slice};
 
 use super::diagnostics;
 use super::macro_parser::{NamedMatches, NamedParseResult};
 
+use rustc_middle::expand::{CanRetry, TcxMacroExpander};
+
 pub(crate) struct ParserAnyMacro<'a> {
     parser: Parser<'a>,
 
@@ -53,6 +56,18 @@ pub(crate) struct ParserAnyMacro<'a> {
 }
 
 impl<'a> ParserAnyMacro<'a> {
+    pub(crate) fn new(
+        parser: Parser<'a>,
+        site_span: Span,
+        macro_ident: Ident,
+        lint_node_id: NodeId,
+        is_trailing_mac: bool,
+        arm_span: Span,
+        is_local: bool,
+    ) -> Self {
+        Self { parser, site_span, macro_ident, lint_node_id, is_trailing_mac, arm_span, is_local }
+    }
+
     pub(crate) fn make(mut self: Box<ParserAnyMacro<'a>>, kind: AstFragmentKind) -> AstFragment {
         let ParserAnyMacro {
             site_span,
@@ -127,6 +142,54 @@ impl TTMacroExpander for MacroRulesMacroExpander {
     }
 }
 
+impl TcxMacroExpander for MacroRulesMacroExpander {
+    fn expand(
+        &self,
+        sess: &Session,
+        sp: Span,
+        input: TokenStream,
+        expand_id: LocalExpnId,
+    ) -> Result<(TokenStream, usize), CanRetry> {
+        // Track nothing for the best performance.
+        let try_success_result =
+            try_match_macro(&sess.psess, self.name, &input, &self.lhses, &mut NoopTracker);
+
+        match try_success_result {
+            Ok((i, named_matches)) => {
+                let (rhs, rhs_span): (&mbe::Delimited, DelimSpan) = match &self.rhses[i] {
+                    mbe::TokenTree::Delimited(span, _, delimited) => (&delimited, *span),
+                    _ => sess.dcx().span_bug(sp, "malformed macro rhs"),
+                };
+
+                // rhs has holes ( `$id` and `$(...)` that need filled)
+                match transcribe(
+                    &sess.psess,
+                    &named_matches,
+                    rhs,
+                    rhs_span,
+                    self.transparency,
+                    expand_id,
+                ) {
+                    Ok(tts) => Ok((tts, i)),
+                    Err(err) => Err(CanRetry::No(err.emit())),
+                }
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    fn name(&self) -> Ident {
+        self.name
+    }
+
+    fn arm_span(&self, rhs: usize) -> Span {
+        self.rhses[rhs].span()
+    }
+
+    fn node_id(&self) -> NodeId {
+        self.node_id
+    }
+}
 struct DummyExpander(ErrorGuaranteed);
 
 impl TTMacroExpander for DummyExpander {
@@ -140,7 +203,11 @@ impl TTMacroExpander for DummyExpander {
     }
 }
 
-fn trace_macros_note(cx_expansions: &mut FxIndexMap<Span, Vec<String>>, sp: Span, message: String) {
+pub(crate) fn trace_macros_note(
+    cx_expansions: &mut FxIndexMap<Span, Vec<String>>,
+    sp: Span,
+    message: String,
+) {
     let sp = sp.macro_backtrace().last().map_or(sp, |trace| trace.call_site);
     cx_expansions.entry(sp).or_default().push(message);
 }
@@ -224,7 +291,14 @@ fn expand_macro<'cx>(
             let arm_span = rhses[i].span();
 
             // rhs has holes ( `$id` and `$(...)` that need filled)
-            let tts = match transcribe(cx, &named_matches, rhs, rhs_span, transparency) {
+            let tts = match transcribe(
+                &cx.sess.psess,
+                &named_matches,
+                rhs,
+                rhs_span,
+                transparency,
+                cx.current_expansion.id,
+            ) {
                 Ok(tts) => tts,
                 Err(err) => {
                     let guar = err.emit();
@@ -270,12 +344,6 @@ fn expand_macro<'cx>(
     }
 }
 
-pub(super) enum CanRetry {
-    Yes,
-    /// We are not allowed to retry macro expansion as a fatal error has been emitted already.
-    No(ErrorGuaranteed),
-}
-
 /// Try expanding the macro. Returns the index of the successful arm and its named_matches if it was successful,
 /// and nothing if it failed. On failure, it's the callers job to use `track` accordingly to record all errors
 /// correctly.
@@ -382,7 +450,19 @@ pub fn compile_declarative_macro(
         )
     };
     let dummy_syn_ext = |guar| (mk_syn_ext(Box::new(DummyExpander(guar))), Vec::new());
-
+    let mk_tcx_syn_ext = |expander| {
+        SyntaxExtension::new(
+            sess,
+            features,
+            SyntaxExtensionKind::TcxLegacyBang(expander),
+            def.span,
+            Vec::new(),
+            edition,
+            def.ident.name,
+            &def.attrs,
+            def.id != DUMMY_NODE_ID,
+        )
+    };
     let dcx = &sess.psess.dcx;
     let lhs_nm = Ident::new(sym::lhs, def.span);
     let rhs_nm = Ident::new(sym::rhs, def.span);
@@ -595,15 +675,15 @@ pub fn compile_declarative_macro(
         })
         .collect();
 
-    let expander = Box::new(MacroRulesMacroExpander {
+    let expander = Lrc::new(MacroRulesMacroExpander {
         name: def.ident,
         span: def.span,
         node_id: def.id,
         transparency,
         lhses,
         rhses,
     });
-    (mk_syn_ext(expander), rule_spans)
+    (mk_tcx_syn_ext(expander), rule_spans)
 }
 
 fn check_lhs_nt_follows(