From 16ae355b480d2a5e12cd6c1bf38925d9bc4879d1 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 22 Apr 2024 16:29:27 +1000 Subject: [PATCH 1/2] Remove a `Span` from `TokenKind::Interpolated`. This span records the declaration of the metavariable in the LHS of the macro. It's used in a couple of error messages. Unfortunately, it gets in the way of the long-term goal of removing `TokenKind::Interpolated`. So this commit removes it, which degrades a couple of (obscure) error messages but makes things simpler and enables the next commit. --- compiler/rustc_ast/src/attr/mod.rs | 2 +- compiler/rustc_ast/src/mut_visit.rs | 2 -- compiler/rustc_ast/src/token.rs | 33 +++++++++++-------- compiler/rustc_ast/src/tokenstream.rs | 6 ++-- compiler/rustc_ast_pretty/src/pprust/state.rs | 2 +- compiler/rustc_expand/src/mbe/diagnostics.rs | 6 ---- compiler/rustc_expand/src/mbe/macro_parser.rs | 11 ++----- compiler/rustc_expand/src/proc_macro.rs | 2 +- .../rustc_expand/src/proc_macro_server.rs | 6 ++-- compiler/rustc_parse/src/parser/attr.rs | 2 +- .../rustc_parse/src/parser/diagnostics.rs | 23 +++++-------- compiler/rustc_parse/src/parser/expr.rs | 2 +- compiler/rustc_parse/src/parser/item.rs | 2 +- compiler/rustc_parse/src/parser/mod.rs | 28 +++++----------- .../rustc_parse/src/parser/nonterminal.rs | 18 +++++----- compiler/rustc_parse/src/parser/pat.rs | 2 +- compiler/rustc_parse/src/parser/path.rs | 2 +- tests/ui/macros/nonterminal-matching.stderr | 2 -- tests/ui/macros/trace_faulty_macros.stderr | 8 ++--- 19 files changed, 62 insertions(+), 97 deletions(-) diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 5e3fc7e335775..7bdbdc4f49f4a 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -342,7 +342,7 @@ impl MetaItem { let span = span.with_hi(segments.last().unwrap().ident.span.hi()); Path { span, segments, tokens: None } } - Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &nt.0 { + Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &**nt { token::Nonterminal::NtMeta(item) => return item.meta(item.path.span), token::Nonterminal::NtPath(path) => (**path).clone(), _ => return None, diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index c4e49d7dbea44..7be5ba82810f4 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -785,8 +785,6 @@ pub fn visit_token(t: &mut Token, vis: &mut T) { } token::Interpolated(nt) => { let nt = Lrc::make_mut(nt); - let (nt, sp) = (&mut nt.0, &mut nt.1); - vis.visit_span(sp); visit_nonterminal(nt, vis); } _ => {} diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index dcdd44c604166..94769a8c345e4 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -111,7 +111,7 @@ impl Lit { Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)), Literal(token_lit) => Some(token_lit), Interpolated(ref nt) - if let NtExpr(expr) | NtLiteral(expr) = &nt.0 + if let NtExpr(expr) | NtLiteral(expr) = &**nt && let ast::ExprKind::Lit(token_lit) = expr.kind => { Some(token_lit) @@ -333,7 +333,11 @@ pub enum TokenKind { /// - It prevents `Token` from implementing `Copy`. /// It adds complexity and likely slows things down. Please don't add new /// occurrences of this token kind! - Interpolated(Lrc<(Nonterminal, Span)>), + /// + /// The span in the surrounding `Token` is that of the metavariable in the + /// macro's RHS. The span within the Nonterminal is that of the fragment + /// passed to the macro at the call site. + Interpolated(Lrc), /// A doc comment token. /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc) @@ -441,7 +445,7 @@ impl Token { /// if they keep spans or perform edition checks. pub fn uninterpolated_span(&self) -> Span { match &self.kind { - Interpolated(nt) => nt.0.use_span(), + Interpolated(nt) => nt.use_span(), _ => self.span, } } @@ -486,7 +490,7 @@ impl Token { PathSep | // global path Lifetime(..) | // labeled loop Pound => true, // expression attributes - Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) | + Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) | NtExpr(..) | NtBlock(..) | NtPath(..)), @@ -510,7 +514,7 @@ impl Token { | DotDot | DotDotDot | DotDotEq // ranges | Lt | BinOp(Shl) // associated path | PathSep => true, // global path - Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) | + Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) | NtPat(..) | NtBlock(..) | NtPath(..)), @@ -533,7 +537,7 @@ impl Token { Lifetime(..) | // lifetime bound in trait object Lt | BinOp(Shl) | // associated path PathSep => true, // global path - Interpolated(ref nt) => matches!(&nt.0, NtTy(..) | NtPath(..)), + Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)), // For anonymous structs or unions, which only appear in specific positions // (type of struct fields or union fields), we don't consider them as regular types _ => false, @@ -544,7 +548,7 @@ impl Token { pub fn can_begin_const_arg(&self) -> bool { match self.kind { OpenDelim(Delimiter::Brace) => true, - Interpolated(ref nt) => matches!(&nt.0, NtExpr(..) | NtBlock(..) | NtLiteral(..)), + Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)), _ => self.can_begin_literal_maybe_minus(), } } @@ -589,7 +593,7 @@ impl Token { match self.uninterpolate().kind { Literal(..) | BinOp(Minus) => true, Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true, - Interpolated(ref nt) => match &nt.0 { + Interpolated(ref nt) => match &**nt { NtLiteral(_) => true, NtExpr(e) => match &e.kind { ast::ExprKind::Lit(_) => true, @@ -610,7 +614,7 @@ impl Token { /// otherwise returns the original token. pub fn uninterpolate(&self) -> Cow<'_, Token> { match &self.kind { - Interpolated(nt) => match &nt.0 { + Interpolated(nt) => match &**nt { NtIdent(ident, is_raw) => { Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span)) } @@ -627,7 +631,7 @@ impl Token { // We avoid using `Token::uninterpolate` here because it's slow. match &self.kind { &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), - Interpolated(nt) => match &nt.0 { + Interpolated(nt) => match &**nt { NtIdent(ident, is_raw) => Some((*ident, *is_raw)), _ => None, }, @@ -641,7 +645,7 @@ impl Token { // We avoid using `Token::uninterpolate` here because it's slow. match &self.kind { &Lifetime(name) => Some(Ident::new(name, self.span)), - Interpolated(nt) => match &nt.0 { + Interpolated(nt) => match &**nt { NtLifetime(ident) => Some(*ident), _ => None, }, @@ -668,7 +672,7 @@ impl Token { /// Returns `true` if the token is an interpolated path. fn is_whole_path(&self) -> bool { if let Interpolated(nt) = &self.kind - && let NtPath(..) = &nt.0 + && let NtPath(..) = &**nt { return true; } @@ -681,7 +685,7 @@ impl Token { /// (which happens while parsing the result of macro expansion)? pub fn is_whole_expr(&self) -> bool { if let Interpolated(nt) = &self.kind - && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = &nt.0 + && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = &**nt { return true; } @@ -692,7 +696,7 @@ impl Token { /// Is the token an interpolated block (`$b:block`)? pub fn is_whole_block(&self) -> bool { if let Interpolated(nt) = &self.kind - && let NtBlock(..) = &nt.0 + && let NtBlock(..) = &**nt { return true; } @@ -857,6 +861,7 @@ pub enum Nonterminal { NtPat(P), NtExpr(P), NtTy(P), + /// The span is for the identifier argument passed to the macro. NtIdent(Ident, IdentIsRaw), NtLifetime(Ident), NtLiteral(P), diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 880f92bbe7bdc..93bf2508a8127 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -491,14 +491,14 @@ impl TokenStream { fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { match &token.kind { - token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = nt.0 => { - TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) + token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = &**nt => { + TokenTree::Token(Token::new(token::Ident(ident.name, *is_raw), ident.span), spacing) } token::Interpolated(nt) => TokenTree::Delimited( DelimSpan::from_single(token.span), DelimSpacing::new(Spacing::JointHidden, spacing), Delimiter::Invisible, - TokenStream::from_nonterminal_ast(&nt.0).flattened(), + TokenStream::from_nonterminal_ast(&nt).flattened(), ), _ => TokenTree::Token(token.clone(), spacing), } diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 2c176828c841f..f438cee4cd252 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -926,7 +926,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere } token::Eof => "".into(), - token::Interpolated(ref nt) => self.nonterminal_to_string(&nt.0).into(), + token::Interpolated(ref nt) => self.nonterminal_to_string(&nt).into(), } } diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs index 15193298ccaa2..8dde76831d0f3 100644 --- a/compiler/rustc_expand/src/mbe/diagnostics.rs +++ b/compiler/rustc_expand/src/mbe/diagnostics.rs @@ -70,12 +70,6 @@ pub(super) fn failed_to_match_macro<'cx>( && (matches!(expected_token.kind, TokenKind::Interpolated(_)) || matches!(token.kind, TokenKind::Interpolated(_))) { - if let TokenKind::Interpolated(node) = &expected_token.kind { - err.span_label(node.1, ""); - } - if let TokenKind::Interpolated(node) = &token.kind { - err.span_label(node.1, ""); - } err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens"); err.note("see for more information"); diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index ffb50f4c92e14..27cf6fee702a7 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -75,10 +75,9 @@ pub(crate) use ParseResult::*; use crate::mbe::{macro_rules::Tracker, KleeneOp, TokenTree}; -use rustc_ast::token::{self, DocComment, Nonterminal, NonterminalKind, Token}; +use rustc_ast::token::{self, DocComment, NonterminalKind, Token}; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; use rustc_errors::ErrorGuaranteed; use rustc_lint_defs::pluralize; use rustc_parse::parser::{ParseNtResult, Parser}; @@ -392,7 +391,7 @@ pub(super) fn count_metavar_decls(matcher: &[TokenTree]) -> usize { #[derive(Debug, Clone)] pub(crate) enum NamedMatch { MatchedSeq(Vec), - MatchedSingle(ParseNtResult>), + MatchedSingle(ParseNtResult), } /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) @@ -686,11 +685,7 @@ impl TtParser { } Ok(nt) => nt, }; - mp.push_match( - next_metavar, - seq_depth, - MatchedSingle(nt.map_nt(|nt| (Lrc::new((nt, span))))), - ); + mp.push_match(next_metavar, seq_depth, MatchedSingle(nt)); mp.idx += 1; } else { unreachable!() diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 4b5c148cb555d..530059e53c210 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -127,7 +127,7 @@ impl MultiItemModifier for DeriveProcMacro { Annotatable::Stmt(stmt) => token::NtStmt(stmt), _ => unreachable!(), }; - TokenStream::token_alone(token::Interpolated(Lrc::new((nt, span))), DUMMY_SP) + TokenStream::token_alone(token::Interpolated(Lrc::new(nt)), DUMMY_SP) } else { item.to_tokens() }; diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 5a66b0fbdef10..8cf9658016182 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -259,7 +259,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { + Interpolated(ref nt) if let NtIdent(ident, is_raw) = &**nt => { trees.push(TokenTree::Ident(Ident { sym: ident.name, is_raw: matches!(is_raw, IdentIsRaw::Yes), @@ -268,14 +268,14 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { - let stream = TokenStream::from_nonterminal_ast(&nt.0); + let stream = TokenStream::from_nonterminal_ast(&nt); // A hack used to pass AST fragments to attribute and derive // macros as a single nonterminal token instead of a token // stream. Such token needs to be "unwrapped" and not // represented as a delimited group. // FIXME: It needs to be removed, but there are some // compatibility issues (see #73345). - if crate::base::nt_pretty_printing_compatibility_hack(&nt.0, rustc.ecx.sess) { + if crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.ecx.sess) { trees.extend(Self::from_internal((stream, rustc))); } else { trees.push(TokenTree::Group(Group { diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index ab5f51eedc307..8e69129559a86 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -365,7 +365,7 @@ impl<'a> Parser<'a> { // We can't use `maybe_whole` here because it would bump in the `None` // case, which we don't want. if let token::Interpolated(nt) = &self.token.kind - && let token::NtMeta(attr_item) = &nt.0 + && let token::NtMeta(attr_item) = &**nt { match attr_item.meta(attr_item.path.span) { Some(meta) => { diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index f256dbf436021..95cec42944ac5 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -2369,9 +2369,9 @@ impl<'a> Parser<'a> { // in a subsequent macro invocation (#71039). let mut tok = self.token.clone(); let mut labels = vec![]; - while let TokenKind::Interpolated(node) = &tok.kind { - let tokens = node.0.tokens(); - labels.push(node.clone()); + while let TokenKind::Interpolated(nt) = &tok.kind { + let tokens = nt.tokens(); + labels.push(nt.clone()); if let Some(tokens) = tokens && let tokens = tokens.to_attr_token_stream() && let tokens = tokens.0.deref() @@ -2384,27 +2384,20 @@ impl<'a> Parser<'a> { } let mut iter = labels.into_iter().peekable(); let mut show_link = false; - while let Some(node) = iter.next() { - let descr = node.0.descr(); + while let Some(nt) = iter.next() { + let descr = nt.descr(); if let Some(next) = iter.peek() { - let next_descr = next.0.descr(); + let next_descr = next.descr(); if next_descr != descr { - err.span_label(next.1, format!("this macro fragment matcher is {next_descr}")); - err.span_label(node.1, format!("this macro fragment matcher is {descr}")); + err.span_label(next.use_span(), format!("this is expected to be {next_descr}")); err.span_label( - next.0.use_span(), - format!("this is expected to be {next_descr}"), - ); - err.span_label( - node.0.use_span(), + nt.use_span(), format!( "this is interpreted as {}, but it is expected to be {}", next_descr, descr, ), ); show_link = true; - } else { - err.span_label(node.1, ""); } } } diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 00947a4c5853f..128659d6b44ca 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -45,7 +45,7 @@ use thin_vec::{thin_vec, ThinVec}; macro_rules! maybe_whole_expr { ($p:expr) => { if let token::Interpolated(nt) = &$p.token.kind { - match &nt.0 { + match &**nt { token::NtExpr(e) | token::NtLiteral(e) => { let e = e.clone(); $p.bump(); diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 84ecd0a0de54f..19a6847533c4a 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -2835,7 +2835,7 @@ impl<'a> Parser<'a> { fn is_named_param(&self) -> bool { let offset = match &self.token.kind { - token::Interpolated(nt) => match &nt.0 { + token::Interpolated(nt) => match &**nt { token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon), _ => 0, }, diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 4a996f89a9a3d..796f8c92f2c6d 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -11,7 +11,6 @@ mod stmt; mod ty; use crate::lexer::UnmatchedDelim; -use ast::token::IdentIsRaw; pub use attr_wrapper::AttrWrapper; pub use diagnostics::AttemptLocalParseRecovery; pub(crate) use expr::ForbiddenLetReason; @@ -20,7 +19,7 @@ pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma}; pub use path::PathStyle; use rustc_ast::ptr::P; -use rustc_ast::token::{self, Delimiter, Token, TokenKind}; +use rustc_ast::token::{self, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind}; use rustc_ast::tokenstream::{AttributesData, DelimSpacing, DelimSpan, Spacing}; use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor}; use rustc_ast::util::case::Case; @@ -31,6 +30,7 @@ use rustc_ast::{ }; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; use rustc_errors::PResult; use rustc_errors::{Applicability, Diag, FatalError, MultiSpan}; use rustc_session::parse::ParseSess; @@ -93,7 +93,7 @@ pub enum TrailingToken { macro_rules! maybe_whole { ($p:expr, $constructor:ident, |$x:ident| $e:expr) => { if let token::Interpolated(nt) = &$p.token.kind - && let token::$constructor(x) = &nt.0 + && let token::$constructor(x) = &**nt { #[allow(unused_mut)] let mut $x = x.clone(); @@ -111,7 +111,7 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath { && $self.may_recover() && $self.look_ahead(1, |t| t == &token::PathSep) && let token::Interpolated(nt) = &$self.token.kind - && let token::NtTy(ty) = &nt.0 + && let token::NtTy(ty) = &**nt { let ty = ty.clone(); $self.bump(); @@ -405,7 +405,7 @@ pub(super) fn token_descr(token: &Token) -> String { (Some(TokenDescription::Keyword), _) => Some("keyword"), (Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"), (Some(TokenDescription::DocComment), _) => Some("doc comment"), - (None, TokenKind::Interpolated(node)) => Some(node.0.descr()), + (None, TokenKind::Interpolated(node)) => Some(node.descr()), (None, _) => None, }; @@ -707,7 +707,7 @@ impl<'a> Parser<'a> { fn check_inline_const(&self, dist: usize) -> bool { self.is_keyword_ahead(dist, &[kw::Const]) && self.look_ahead(dist + 1, |t| match &t.kind { - token::Interpolated(nt) => matches!(&nt.0, token::NtBlock(..)), + token::Interpolated(nt) => matches!(&**nt, token::NtBlock(..)), token::OpenDelim(Delimiter::Brace) => true, _ => false, }) @@ -1591,19 +1591,7 @@ pub enum FlatToken { // Metavar captures of various kinds. #[derive(Clone, Debug)] -pub enum ParseNtResult { +pub enum ParseNtResult { Tt(TokenTree), - Nt(NtType), -} - -impl ParseNtResult { - pub fn map_nt(self, mut f: F) -> ParseNtResult - where - F: FnMut(T) -> U, - { - match self { - ParseNtResult::Tt(tt) => ParseNtResult::Tt(tt), - ParseNtResult::Nt(nt) => ParseNtResult::Nt(f(nt)), - } - } + Nt(Lrc), } diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 73b17353ac90c..3a560382d327c 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -1,7 +1,8 @@ use rustc_ast::ptr::P; -use rustc_ast::token::{self, Delimiter, Nonterminal, Nonterminal::*, NonterminalKind, Token}; +use rustc_ast::token::{self, Delimiter, Nonterminal::*, NonterminalKind, Token}; use rustc_ast::HasTokens; use rustc_ast_pretty::pprust; +use rustc_data_structures::sync::Lrc; use rustc_errors::PResult; use rustc_span::symbol::{kw, Ident}; @@ -54,7 +55,7 @@ impl<'a> Parser<'a> { }, NonterminalKind::Block => match &token.kind { token::OpenDelim(Delimiter::Brace) => true, - token::Interpolated(nt) => match &nt.0 { + token::Interpolated(nt) => match &**nt { NtBlock(_) | NtLifetime(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true, NtItem(_) | NtPat(_) | NtTy(_) | NtIdent(..) | NtMeta(_) | NtPath(_) | NtVis(_) => false, @@ -63,7 +64,7 @@ impl<'a> Parser<'a> { }, NonterminalKind::Path | NonterminalKind::Meta => match &token.kind { token::PathSep | token::Ident(..) => true, - token::Interpolated(nt) => may_be_ident(&nt.0), + token::Interpolated(nt) => may_be_ident(nt), _ => false, }, NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind { @@ -81,13 +82,13 @@ impl<'a> Parser<'a> { token::BinOp(token::Shl) => true, // path (double UFCS) // leading vert `|` or-pattern token::BinOp(token::Or) => matches!(kind, NonterminalKind::PatWithOr), - token::Interpolated(nt) => may_be_ident(&nt.0), + token::Interpolated(nt) => may_be_ident(nt), _ => false, }, NonterminalKind::Lifetime => match &token.kind { token::Lifetime(_) => true, token::Interpolated(nt) => { - matches!(&nt.0, NtLifetime(_)) + matches!(&**nt, NtLifetime(_)) } _ => false, }, @@ -100,10 +101,7 @@ impl<'a> Parser<'a> { /// Parse a non-terminal (e.g. MBE `:pat` or `:ident`). Inlined because there is only one call /// site. #[inline] - pub fn parse_nonterminal( - &mut self, - kind: NonterminalKind, - ) -> PResult<'a, ParseNtResult> { + pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, ParseNtResult> { // A `macro_rules!` invocation may pass a captured item/expr to a proc-macro, // which requires having captured tokens available. Since we cannot determine // in advance whether or not a proc-macro will be (transitively) invoked, @@ -196,7 +194,7 @@ impl<'a> Parser<'a> { ); } - Ok(ParseNtResult::Nt(nt)) + Ok(ParseNtResult::Nt(Lrc::new(nt))) } } diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs index 78d3d019bf4d2..8af415f7c9dd3 100644 --- a/compiler/rustc_parse/src/parser/pat.rs +++ b/compiler/rustc_parse/src/parser/pat.rs @@ -757,7 +757,7 @@ impl<'a> Parser<'a> { // Make sure we don't allow e.g. `let mut $p;` where `$p:pat`. if let token::Interpolated(nt) = &self.token.kind { - if let token::NtPat(..) = &nt.0 { + if let token::NtPat(..) = &**nt { self.expected_ident_found_err().emit(); } } diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs index 0f410772dd980..c6d64ff15814e 100644 --- a/compiler/rustc_parse/src/parser/path.rs +++ b/compiler/rustc_parse/src/parser/path.rs @@ -187,7 +187,7 @@ impl<'a> Parser<'a> { }); if let token::Interpolated(nt) = &self.token.kind { - if let token::NtTy(ty) = &nt.0 { + if let token::NtTy(ty) = &**nt { if let ast::TyKind::Path(None, path) = &ty.kind { let path = path.clone(); self.bump(); diff --git a/tests/ui/macros/nonterminal-matching.stderr b/tests/ui/macros/nonterminal-matching.stderr index 499f9a7637d22..d19141145fa19 100644 --- a/tests/ui/macros/nonterminal-matching.stderr +++ b/tests/ui/macros/nonterminal-matching.stderr @@ -1,8 +1,6 @@ error: no rules expected the token `enum E {}` --> $DIR/nonterminal-matching.rs:19:10 | -LL | macro complex_nonterminal($nt_item: item) { - | -------------- LL | macro n(a $nt_item b) { | --------------------- when calling this macro ... diff --git a/tests/ui/macros/trace_faulty_macros.stderr b/tests/ui/macros/trace_faulty_macros.stderr index 69607600b55c8..665dcc7d0913c 100644 --- a/tests/ui/macros/trace_faulty_macros.stderr +++ b/tests/ui/macros/trace_faulty_macros.stderr @@ -73,14 +73,10 @@ error: expected expression, found pattern `1 + 1` --> $DIR/trace_faulty_macros.rs:49:37 | LL | (let $p:pat = $e:expr) => {test!(($p,$e))}; - | ------- -- this is interpreted as expression, but it is expected to be pattern - | | - | this macro fragment matcher is expression + | -- this is interpreted as expression, but it is expected to be pattern ... LL | (($p:pat, $e:pat)) => {let $p = $e;}; - | ------ ^^ expected expression - | | - | this macro fragment matcher is pattern + | ^^ expected expression ... LL | test!(let x = 1+1); | ------------------ From beced6741bfe50b4d1b16707e79f15b10532f996 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 22 Apr 2024 19:46:51 +1000 Subject: [PATCH 2/2] Remove `NtIdent` and `NtLifetime`. The extra span is now recorded in the new `TokenKind::InterpolatedIdent` and `TokenKind::InterpolatedLifetime`. These both consist of a single token, and so there's no operator precedence problems with inserting them directly into the token stream. The other way to do this would be to wrap the ident/lifetime in invisible delimiters, but there's a lot of code that assumes an interpolated ident/lifetime fits in a single token, and changing all that code to work with invisible delimiters would have been a pain. (Maybe it could be done in a follow-up.) This change might not seem like much of a win, but it's a first step toward the much bigger and long-desired removal of `Nonterminal` and `TokenKind::Interpolated`. That change is big and complex enough that it's worth doing this piece separately. (Indeed, this commit is based on part of a late commit in #114647, a prior attempt at that big and complex change.) --- compiler/rustc_ast/src/ast_traits.rs | 2 - compiler/rustc_ast/src/mut_visit.rs | 5 +- compiler/rustc_ast/src/token.rs | 121 ++++++++++-------- compiler/rustc_ast/src/tokenstream.rs | 27 ++-- compiler/rustc_ast_pretty/src/pprust/state.rs | 14 +- compiler/rustc_expand/src/config.rs | 15 ++- compiler/rustc_expand/src/mbe/transcribe.rs | 13 +- .../rustc_expand/src/proc_macro_server.rs | 23 ++-- compiler/rustc_parse/src/parser/expr.rs | 4 +- compiler/rustc_parse/src/parser/mod.rs | 6 + .../rustc_parse/src/parser/nonterminal.rs | 53 ++++---- 11 files changed, 171 insertions(+), 112 deletions(-) diff --git a/compiler/rustc_ast/src/ast_traits.rs b/compiler/rustc_ast/src/ast_traits.rs index a0486227f2afb..2cf811e9122f6 100644 --- a/compiler/rustc_ast/src/ast_traits.rs +++ b/compiler/rustc_ast/src/ast_traits.rs @@ -240,7 +240,6 @@ impl HasTokens for Nonterminal { Nonterminal::NtPath(path) => path.tokens(), Nonterminal::NtVis(vis) => vis.tokens(), Nonterminal::NtBlock(block) => block.tokens(), - Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None, } } fn tokens_mut(&mut self) -> Option<&mut Option> { @@ -254,7 +253,6 @@ impl HasTokens for Nonterminal { Nonterminal::NtPath(path) => path.tokens_mut(), Nonterminal::NtVis(vis) => vis.tokens_mut(), Nonterminal::NtBlock(block) => block.tokens_mut(), - Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None, } } } diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 7be5ba82810f4..e5fb3451250a4 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -783,6 +783,9 @@ pub fn visit_token(t: &mut Token, vis: &mut T) { *span = ident.span; return; // Avoid visiting the span for the second time. } + token::InterpolatedIdent(ident, _) | token::InterpolatedLifetime(ident) => { + vis.visit_ident(ident); + } token::Interpolated(nt) => { let nt = Lrc::make_mut(nt); visit_nonterminal(nt, vis); @@ -834,8 +837,6 @@ pub fn visit_nonterminal(nt: &mut token::Nonterminal, vis: &mut T token::NtPat(pat) => vis.visit_pat(pat), token::NtExpr(expr) => vis.visit_expr(expr), token::NtTy(ty) => vis.visit_ty(ty), - token::NtIdent(ident, _is_raw) => vis.visit_ident(ident), - token::NtLifetime(ident) => vis.visit_ident(ident), token::NtLiteral(expr) => vis.visit_expr(expr), token::NtMeta(item) => { let AttrItem { path, args, tokens } = item.deref_mut(); diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 94769a8c345e4..e7e345e61bc42 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -314,15 +314,24 @@ pub enum TokenKind { Literal(Lit), /// Identifier token. - /// Do not forget about `NtIdent` when you want to match on identifiers. + /// Do not forget about `InterpolatedIdent` when you want to match on identifiers. /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to /// treat regular and interpolated identifiers in the same way. Ident(Symbol, IdentIsRaw), + /// This identifier (and its span) is the identifier passed to the + /// declarative macro. The span in the surrounding `Token` is the span of + /// the `ident` metavariable in the macro's RHS. + InterpolatedIdent(Ident, IdentIsRaw), + /// Lifetime identifier token. - /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. + /// Do not forget about `InterpolatedLIfetime` when you want to match on lifetime identifiers. /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to /// treat regular and interpolated lifetime identifiers in the same way. Lifetime(Symbol), + /// This identifier (and its span) is the lifetime passed to the + /// declarative macro. The span in the surrounding `Token` is the span of + /// the `lifetime` metavariable in the macro's RHS. + InterpolatedLifetime(Ident), /// An embedded AST node, as produced by a macro. This only exists for /// historical reasons. We'd like to get rid of it, for multiple reasons. @@ -444,8 +453,9 @@ impl Token { /// Note that keywords are also identifiers, so they should use this /// if they keep spans or perform edition checks. pub fn uninterpolated_span(&self) -> Span { - match &self.kind { - Interpolated(nt) => nt.use_span(), + match self.kind { + InterpolatedIdent(ident, _) | InterpolatedLifetime(ident) => ident.span, + Interpolated(ref nt) => nt.use_span(), _ => self.span, } } @@ -462,8 +472,16 @@ impl Token { true } - OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | Eof => false, + OpenDelim(..) + | CloseDelim(..) + | Literal(..) + | DocComment(..) + | Ident(..) + | InterpolatedIdent(..) + | Lifetime(..) + | InterpolatedLifetime(..) + | Interpolated(..) + | Eof => false, } } @@ -613,14 +631,11 @@ impl Token { /// into the regular identifier or lifetime token it refers to, /// otherwise returns the original token. pub fn uninterpolate(&self) -> Cow<'_, Token> { - match &self.kind { - Interpolated(nt) => match &**nt { - NtIdent(ident, is_raw) => { - Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span)) - } - NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), - _ => Cow::Borrowed(self), - }, + match self.kind { + InterpolatedIdent(ident, is_raw) => { + Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)) + } + InterpolatedLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), _ => Cow::Borrowed(self), } } @@ -629,12 +644,9 @@ impl Token { #[inline] pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> { // We avoid using `Token::uninterpolate` here because it's slow. - match &self.kind { - &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), - Interpolated(nt) => match &**nt { - NtIdent(ident, is_raw) => Some((*ident, *is_raw)), - _ => None, - }, + match self.kind { + Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), + InterpolatedIdent(ident, is_raw) => Some((ident, is_raw)), _ => None, } } @@ -643,12 +655,9 @@ impl Token { #[inline] pub fn lifetime(&self) -> Option { // We avoid using `Token::uninterpolate` here because it's slow. - match &self.kind { - &Lifetime(name) => Some(Ident::new(name, self.span)), - Interpolated(nt) => match &**nt { - NtLifetime(ident) => Some(*ident), - _ => None, - }, + match self.kind { + Lifetime(name) => Some(Ident::new(name, self.span)), + InterpolatedLifetime(ident) => Some(ident), _ => None, } } @@ -835,10 +844,36 @@ impl Token { _ => return None, }, - Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot - | DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar - | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, + Le + | EqEq + | Ne + | Ge + | AndAnd + | OrOr + | Tilde + | BinOpEq(..) + | At + | DotDotDot + | DotDotEq + | Comma + | Semi + | PathSep + | RArrow + | LArrow + | FatArrow + | Pound + | Dollar + | Question + | OpenDelim(..) + | CloseDelim(..) + | Literal(..) + | Ident(..) + | InterpolatedIdent(..) + | Lifetime(..) + | InterpolatedLifetime(..) + | Interpolated(..) + | DocComment(..) + | Eof => return None, }; Some(Token::new(kind, self.span.to(joint.span))) @@ -861,9 +896,6 @@ pub enum Nonterminal { NtPat(P), NtExpr(P), NtTy(P), - /// The span is for the identifier argument passed to the macro. - NtIdent(Ident, IdentIsRaw), - NtLifetime(Ident), NtLiteral(P), /// Stuff inside brackets for attributes NtMeta(P), @@ -958,7 +990,6 @@ impl Nonterminal { NtPat(pat) => pat.span, NtExpr(expr) | NtLiteral(expr) => expr.span, NtTy(ty) => ty.span, - NtIdent(ident, _) | NtLifetime(ident) => ident.span, NtMeta(attr_item) => attr_item.span(), NtPath(path) => path.span, NtVis(vis) => vis.span, @@ -974,8 +1005,6 @@ impl Nonterminal { NtExpr(..) => "expression", NtLiteral(..) => "literal", NtTy(..) => "type", - NtIdent(..) => "identifier", - NtLifetime(..) => "lifetime", NtMeta(..) => "attribute", NtPath(..) => "path", NtVis(..) => "visibility", @@ -984,18 +1013,12 @@ impl Nonterminal { } impl PartialEq for Nonterminal { - fn eq(&self, rhs: &Self) -> bool { - match (self, rhs) { - (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { - ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs - } - (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, - // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them - // correctly based on data from AST. This will prevent them from matching each other - // in macros. The comparison will become possible only when each nonterminal has an - // attached token stream from which it was parsed. - _ => false, - } + fn eq(&self, _rhs: &Self) -> bool { + // FIXME: Assume that all nonterminals are not equal, we can't compare them + // correctly based on data from AST. This will prevent them from matching each other + // in macros. The comparison will become possible only when each nonterminal has an + // attached token stream from which it was parsed. + false } } @@ -1008,12 +1031,10 @@ impl fmt::Debug for Nonterminal { NtPat(..) => f.pad("NtPat(..)"), NtExpr(..) => f.pad("NtExpr(..)"), NtTy(..) => f.pad("NtTy(..)"), - NtIdent(..) => f.pad("NtIdent(..)"), NtLiteral(..) => f.pad("NtLiteral(..)"), NtMeta(..) => f.pad("NtMeta(..)"), NtPath(..) => f.pad("NtPath(..)"), NtVis(..) => f.pad("NtVis(..)"), - NtLifetime(..) => f.pad("NtLifetime(..)"), } } } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 93bf2508a8127..48b84f46bfed8 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -467,12 +467,6 @@ impl TokenStream { pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { match nt { - Nonterminal::NtIdent(ident, is_raw) => { - TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span) - } - Nonterminal::NtLifetime(ident) => { - TokenStream::token_alone(token::Lifetime(ident.name), ident.span) - } Nonterminal::NtItem(item) => TokenStream::from_ast(item), Nonterminal::NtBlock(block) => TokenStream::from_ast(block), Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => { @@ -490,11 +484,17 @@ impl TokenStream { } fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { - match &token.kind { - token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = &**nt => { - TokenTree::Token(Token::new(token::Ident(ident.name, *is_raw), ident.span), spacing) + match token.kind { + token::InterpolatedIdent(ident, is_raw) => { + TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) } - token::Interpolated(nt) => TokenTree::Delimited( + token::InterpolatedLifetime(ident) => TokenTree::Delimited( + DelimSpan::from_single(token.span), + DelimSpacing::new(Spacing::JointHidden, spacing), + Delimiter::Invisible, + TokenStream::token_alone(token::Lifetime(ident.name), ident.span), + ), + token::Interpolated(ref nt) => TokenTree::Delimited( DelimSpan::from_single(token.span), DelimSpacing::new(Spacing::JointHidden, spacing), Delimiter::Invisible, @@ -517,7 +517,12 @@ impl TokenStream { pub fn flattened(&self) -> TokenStream { fn can_skip(stream: &TokenStream) -> bool { stream.trees().all(|tree| match tree { - TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)), + TokenTree::Token(token, _) => !matches!( + token.kind, + token::InterpolatedIdent(..) + | token::InterpolatedLifetime(..) + | token::Interpolated(..) + ), TokenTree::Delimited(.., inner) => can_skip(inner), }) } diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index f438cee4cd252..750a627262797 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -852,8 +852,6 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere token::NtBlock(e) => self.block_to_string(e), token::NtStmt(e) => self.stmt_to_string(e), token::NtPat(e) => self.pat_to_string(e), - &token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw.into()).to_string(), - token::NtLifetime(e) => e.to_string(), token::NtLiteral(e) => self.expr_to_string(e), token::NtVis(e) => self.vis_to_string(e), } @@ -915,10 +913,16 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere token::Literal(lit) => literal_to_string(lit).into(), /* Name components */ - token::Ident(s, is_raw) => { - IdentPrinter::new(s, is_raw.into(), convert_dollar_crate).to_string().into() + token::Ident(name, is_raw) => { + IdentPrinter::new(name, is_raw.into(), convert_dollar_crate).to_string().into() } - token::Lifetime(s) => s.to_string().into(), + token::InterpolatedIdent(ident, is_raw) => { + IdentPrinter::new(ident.name, is_raw.into(), convert_dollar_crate) + .to_string() + .into() + } + token::Lifetime(name) => name.to_string().into(), + token::InterpolatedLifetime(ident) => ident.name.to_string().into(), /* Other */ token::DocComment(comment_kind, attr_style, data) => { diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index c95d7cdeb73ab..ca38e08388866 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -199,10 +199,17 @@ impl<'a> StripUnconfigured<'a> { inner = self.configure_tokens(&inner); Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter() } - AttrTokenTree::Token(ref token, _) - if let TokenKind::Interpolated(nt) = &token.kind => - { - panic!("Nonterminal should have been flattened at {:?}: {:?}", token.span, nt); + AttrTokenTree::Token( + Token { + kind: + TokenKind::InterpolatedIdent(..) + | TokenKind::InterpolatedLifetime(..) + | TokenKind::Interpolated(..), + .. + }, + _, + ) => { + panic!("Nonterminal should have been flattened: {:?}", tree); } AttrTokenTree::Token(token, spacing) => { Some(AttrTokenTree::Token(token, spacing)).into_iter() diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index dad83984c8b15..54dafbc8d0e04 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -256,10 +256,17 @@ pub(super) fn transcribe<'a>( // without wrapping them into groups. maybe_use_metavar_location(cx, &stack, sp, tt, &mut marker) } + MatchedSingle(ParseNtResult::Ident(ident, is_raw)) => { + marker.visit_span(&mut sp); + let kind = token::InterpolatedIdent(*ident, *is_raw); + TokenTree::token_alone(kind, sp) + } + MatchedSingle(ParseNtResult::Lifetime(ident)) => { + marker.visit_span(&mut sp); + let kind = token::InterpolatedLifetime(*ident); + TokenTree::token_alone(kind, sp) + } MatchedSingle(ParseNtResult::Nt(nt)) => { - // Other variables are emitted into the output stream as groups with - // `Delimiter::Invisible` to maintain parsing priorities. - // `Interpolated` is currently used for such groups in rustc parser. marker.visit_span(&mut sp); TokenTree::token_alone(token::Interpolated(nt.clone()), sp) } diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 8cf9658016182..f53143729ae5c 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -220,6 +220,12 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { trees.push(TokenTree::Ident(Ident { sym, is_raw: is_raw.into(), span })) } + InterpolatedIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident { + sym: ident.name, + is_raw: is_raw.into(), + span: ident.span, + })), + Lifetime(name) => { let ident = symbol::Ident::new(name, span).without_first_quote(); trees.extend([ @@ -227,6 +233,15 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { + let stream = TokenStream::token_alone(token::Lifetime(ident.name), ident.span); + trees.push(TokenTree::Group(Group { + delimiter: pm::Delimiter::None, + stream: Some(stream), + span: DelimSpan::from_single(span), + })) + } + Literal(token::Lit { kind, symbol, suffix }) => { trees.push(TokenTree::Literal(self::Literal { kind: FromInternal::from_internal(kind), @@ -259,14 +274,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { - trees.push(TokenTree::Ident(Ident { - sym: ident.name, - is_raw: matches!(is_raw, IdentIsRaw::Yes), - span: ident.span, - })) - } - Interpolated(nt) => { let stream = TokenStream::from_nonterminal_ast(&nt); // A hack used to pass AST fragments to attribute and derive diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 128659d6b44ca..5e09dc4d2dd3c 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -710,7 +710,9 @@ impl<'a> Parser<'a> { /// Returns the span of expr if it was not interpolated, or the span of the interpolated token. fn interpolated_or_expr_span(&self, expr: &Expr) -> Span { match self.prev_token.kind { - TokenKind::Interpolated(..) => self.prev_token.span, + TokenKind::InterpolatedIdent(..) + | TokenKind::InterpolatedLifetime(..) + | TokenKind::Interpolated(..) => self.prev_token.span, _ => expr.span, } } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 796f8c92f2c6d..140718a855d70 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -405,6 +405,8 @@ pub(super) fn token_descr(token: &Token) -> String { (Some(TokenDescription::Keyword), _) => Some("keyword"), (Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"), (Some(TokenDescription::DocComment), _) => Some("doc comment"), + (None, TokenKind::InterpolatedIdent(..)) => Some("identifier"), + (None, TokenKind::InterpolatedLifetime(..)) => Some("lifetime"), (None, TokenKind::Interpolated(node)) => Some(node.descr()), (None, _) => None, }; @@ -1593,5 +1595,9 @@ pub enum FlatToken { #[derive(Clone, Debug)] pub enum ParseNtResult { Tt(TokenTree), + Ident(Ident, IdentIsRaw), + Lifetime(Ident), + + /// This case will eventually be removed, along with `Token::Interpolate`. Nt(Lrc), } diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 3a560382d327c..91ec979a5865f 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -25,15 +25,13 @@ impl<'a> Parser<'a> { | NtPat(_) | NtExpr(_) | NtTy(_) - | NtIdent(..) | NtLiteral(_) // `true`, `false` | NtMeta(_) | NtPath(_) => true, NtItem(_) | NtBlock(_) - | NtVis(_) - | NtLifetime(_) => false, + | NtVis(_) => false, } } @@ -50,25 +48,30 @@ impl<'a> Parser<'a> { NonterminalKind::Literal => token.can_begin_literal_maybe_minus(), NonterminalKind::Vis => match token.kind { // The follow-set of :vis + "priv" keyword + interpolated - token::Comma | token::Ident(..) | token::Interpolated(_) => true, + token::Comma + | token::Ident(..) + | token::InterpolatedIdent(..) + | token::InterpolatedLifetime(..) + | token::Interpolated(_) => true, _ => token.can_begin_type(), }, NonterminalKind::Block => match &token.kind { token::OpenDelim(Delimiter::Brace) => true, + token::InterpolatedLifetime(..) => true, token::Interpolated(nt) => match &**nt { - NtBlock(_) | NtLifetime(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true, - NtItem(_) | NtPat(_) | NtTy(_) | NtIdent(..) | NtMeta(_) | NtPath(_) - | NtVis(_) => false, + NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true, + NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false, }, _ => false, }, NonterminalKind::Path | NonterminalKind::Meta => match &token.kind { - token::PathSep | token::Ident(..) => true, + token::PathSep | token::Ident(..) | token::InterpolatedIdent(..) => true, token::Interpolated(nt) => may_be_ident(nt), _ => false, }, NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind { - token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) + // box, ref, mut, and other identifiers (can stricten) + token::Ident(..) | token::InterpolatedIdent(..) | token::OpenDelim(Delimiter::Parenthesis) | // tuple pattern token::OpenDelim(Delimiter::Bracket) | // slice pattern token::BinOp(token::And) | // reference @@ -86,10 +89,7 @@ impl<'a> Parser<'a> { _ => false, }, NonterminalKind::Lifetime => match &token.kind { - token::Lifetime(_) => true, - token::Interpolated(nt) => { - matches!(&**nt, NtLifetime(_)) - } + token::Lifetime(_) | token::InterpolatedLifetime(..) => true, _ => false, }, NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => { @@ -154,15 +154,16 @@ impl<'a> Parser<'a> { } // this could be handled like a token, since it is one - NonterminalKind::Ident if let Some((ident, is_raw)) = get_macro_ident(&self.token) => { - self.bump(); - NtIdent(ident, is_raw) - } NonterminalKind::Ident => { - return Err(self.dcx().create_err(UnexpectedNonterminal::Ident { - span: self.token.span, - token: self.token.clone(), - })); + return if let Some((ident, is_raw)) = get_macro_ident(&self.token) { + self.bump(); + Ok(ParseNtResult::Ident(ident, is_raw)) + } else { + Err(self.dcx().create_err(UnexpectedNonterminal::Ident { + span: self.token.span, + token: self.token.clone(), + })) + }; } NonterminalKind::Path => { NtPath(P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?)) @@ -173,14 +174,14 @@ impl<'a> Parser<'a> { .collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?)) } NonterminalKind::Lifetime => { - if self.check_lifetime() { - NtLifetime(self.expect_lifetime().ident) + return if self.check_lifetime() { + Ok(ParseNtResult::Lifetime(self.expect_lifetime().ident)) } else { - return Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime { + Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime { span: self.token.span, token: self.token.clone(), - })); - } + })) + }; } };