diff --git a/compiler/rustc_ast/src/ast_traits.rs b/compiler/rustc_ast/src/ast_traits.rs index a0486227f2afb..2cf811e9122f6 100644 --- a/compiler/rustc_ast/src/ast_traits.rs +++ b/compiler/rustc_ast/src/ast_traits.rs @@ -240,7 +240,6 @@ impl HasTokens for Nonterminal { Nonterminal::NtPath(path) => path.tokens(), Nonterminal::NtVis(vis) => vis.tokens(), Nonterminal::NtBlock(block) => block.tokens(), - Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None, } } fn tokens_mut(&mut self) -> Option<&mut Option> { @@ -254,7 +253,6 @@ impl HasTokens for Nonterminal { Nonterminal::NtPath(path) => path.tokens_mut(), Nonterminal::NtVis(vis) => vis.tokens_mut(), Nonterminal::NtBlock(block) => block.tokens_mut(), - Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None, } } } diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 07358047025ab..5c4162295bbc1 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -781,6 +781,12 @@ pub fn visit_token(t: &mut Token, vis: &mut T) { *span = ident.span; return; // Avoid visiting the span for the second time. } + token::NtIdent(ident, _is_raw) => { + vis.visit_ident(ident); + } + token::NtLifetime(ident) => { + vis.visit_ident(ident); + } token::Interpolated(nt) => { let nt = Lrc::make_mut(nt); visit_nonterminal(nt, vis); @@ -832,8 +838,6 @@ fn visit_nonterminal(nt: &mut token::Nonterminal, vis: &mut T) { token::NtPat(pat) => vis.visit_pat(pat), token::NtExpr(expr) => vis.visit_expr(expr), token::NtTy(ty) => vis.visit_ty(ty), - token::NtIdent(ident, _is_raw) => vis.visit_ident(ident), - token::NtLifetime(ident) => vis.visit_ident(ident), token::NtLiteral(expr) => vis.visit_expr(expr), token::NtMeta(item) => { let AttrItem { path, args, tokens } = item.deref_mut(); diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 590c3daf0c223..2712171b57eab 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -318,11 +318,20 @@ pub enum TokenKind { /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to /// treat regular and interpolated identifiers in the same way. Ident(Symbol, IdentIsRaw), + /// This identifier (and its span) is the identifier passed to the + /// declarative macro. The span in the surrounding `Token` is the span of + /// the `ident` metavariable in the macro's RHS. + NtIdent(Ident, IdentIsRaw), + /// Lifetime identifier token. - /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. + /// Do not forget about `InterpolatedLIfetime` when you want to match on lifetime identifiers. /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to /// treat regular and interpolated lifetime identifiers in the same way. Lifetime(Symbol), + /// This identifier (and its span) is the lifetime passed to the + /// declarative macro. The span in the surrounding `Token` is the span of + /// the `lifetime` metavariable in the macro's RHS. + NtLifetime(Ident), /// An embedded AST node, as produced by a macro. This only exists for /// historical reasons. We'd like to get rid of it, for multiple reasons. @@ -444,8 +453,9 @@ impl Token { /// Note that keywords are also identifiers, so they should use this /// if they keep spans or perform edition checks. pub fn uninterpolated_span(&self) -> Span { - match &self.kind { - Interpolated(nt) => nt.use_span(), + match self.kind { + NtIdent(ident, _) | NtLifetime(ident) => ident.span, + Interpolated(ref nt) => nt.use_span(), _ => self.span, } } @@ -463,7 +473,7 @@ impl Token { } OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | Eof => false, + | NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false, } } @@ -613,14 +623,9 @@ impl Token { /// into the regular identifier or lifetime token it refers to, /// otherwise returns the original token. pub fn uninterpolate(&self) -> Cow<'_, Token> { - match &self.kind { - Interpolated(nt) => match &**nt { - NtIdent(ident, is_raw) => { - Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span)) - } - NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), - _ => Cow::Borrowed(self), - }, + match self.kind { + NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)), + NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), _ => Cow::Borrowed(self), } } @@ -629,12 +634,9 @@ impl Token { #[inline] pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> { // We avoid using `Token::uninterpolate` here because it's slow. - match &self.kind { - &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), - Interpolated(nt) => match &**nt { - NtIdent(ident, is_raw) => Some((*ident, *is_raw)), - _ => None, - }, + match self.kind { + Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), + NtIdent(ident, is_raw) => Some((ident, is_raw)), _ => None, } } @@ -643,12 +645,9 @@ impl Token { #[inline] pub fn lifetime(&self) -> Option { // We avoid using `Token::uninterpolate` here because it's slow. - match &self.kind { - &Lifetime(name) => Some(Ident::new(name, self.span)), - Interpolated(nt) => match &**nt { - NtLifetime(ident) => Some(*ident), - _ => None, - }, + match self.kind { + Lifetime(name) => Some(Ident::new(name, self.span)), + NtLifetime(ident) => Some(ident), _ => None, } } @@ -837,8 +836,10 @@ impl Token { Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar - | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, + | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..) + | Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof => { + return None; + } }; Some(Token::new(kind, self.span.to(joint.span))) @@ -861,9 +862,6 @@ pub enum Nonterminal { NtPat(P), NtExpr(P), NtTy(P), - /// The span is for the identifier argument passed to the macro. - NtIdent(Ident, IdentIsRaw), - NtLifetime(Ident), NtLiteral(P), /// Stuff inside brackets for attributes NtMeta(P), @@ -958,7 +956,6 @@ impl Nonterminal { NtPat(pat) => pat.span, NtExpr(expr) | NtLiteral(expr) => expr.span, NtTy(ty) => ty.span, - NtIdent(ident, _) | NtLifetime(ident) => ident.span, NtMeta(attr_item) => attr_item.span(), NtPath(path) => path.span, NtVis(vis) => vis.span, @@ -974,8 +971,6 @@ impl Nonterminal { NtExpr(..) => "expression", NtLiteral(..) => "literal", NtTy(..) => "type", - NtIdent(..) => "identifier", - NtLifetime(..) => "lifetime", NtMeta(..) => "attribute", NtPath(..) => "path", NtVis(..) => "visibility", @@ -984,18 +979,12 @@ impl Nonterminal { } impl PartialEq for Nonterminal { - fn eq(&self, rhs: &Self) -> bool { - match (self, rhs) { - (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { - ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs - } - (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, - // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them - // correctly based on data from AST. This will prevent them from matching each other - // in macros. The comparison will become possible only when each nonterminal has an - // attached token stream from which it was parsed. - _ => false, - } + fn eq(&self, _rhs: &Self) -> bool { + // FIXME: Assume that all nonterminals are not equal, we can't compare them + // correctly based on data from AST. This will prevent them from matching each other + // in macros. The comparison will become possible only when each nonterminal has an + // attached token stream from which it was parsed. + false } } @@ -1008,12 +997,10 @@ impl fmt::Debug for Nonterminal { NtPat(..) => f.pad("NtPat(..)"), NtExpr(..) => f.pad("NtExpr(..)"), NtTy(..) => f.pad("NtTy(..)"), - NtIdent(..) => f.pad("NtIdent(..)"), NtLiteral(..) => f.pad("NtLiteral(..)"), NtMeta(..) => f.pad("NtMeta(..)"), NtPath(..) => f.pad("NtPath(..)"), NtVis(..) => f.pad("NtVis(..)"), - NtLifetime(..) => f.pad("NtLifetime(..)"), } } } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index d532a884184ce..fb666550e9302 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -466,12 +466,6 @@ impl TokenStream { pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { match nt { - Nonterminal::NtIdent(ident, is_raw) => { - TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span) - } - Nonterminal::NtLifetime(ident) => { - TokenStream::token_alone(token::Lifetime(ident.name), ident.span) - } Nonterminal::NtItem(item) => TokenStream::from_ast(item), Nonterminal::NtBlock(block) => TokenStream::from_ast(block), Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => { @@ -489,11 +483,17 @@ impl TokenStream { } fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { - match &token.kind { - token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = &**nt => { - TokenTree::Token(Token::new(token::Ident(ident.name, *is_raw), ident.span), spacing) + match token.kind { + token::NtIdent(ident, is_raw) => { + TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) } - token::Interpolated(nt) => TokenTree::Delimited( + token::NtLifetime(ident) => TokenTree::Delimited( + DelimSpan::from_single(token.span), + DelimSpacing::new(Spacing::JointHidden, spacing), + Delimiter::Invisible, + TokenStream::token_alone(token::Lifetime(ident.name), ident.span), + ), + token::Interpolated(ref nt) => TokenTree::Delimited( DelimSpan::from_single(token.span), DelimSpacing::new(Spacing::JointHidden, spacing), Delimiter::Invisible, @@ -516,7 +516,10 @@ impl TokenStream { pub fn flattened(&self) -> TokenStream { fn can_skip(stream: &TokenStream) -> bool { stream.trees().all(|tree| match tree { - TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)), + TokenTree::Token(token, _) => !matches!( + token.kind, + token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..) + ), TokenTree::Delimited(.., inner) => can_skip(inner), }) } diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index f438cee4cd252..e9cce4b3c3449 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -852,8 +852,6 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere token::NtBlock(e) => self.block_to_string(e), token::NtStmt(e) => self.stmt_to_string(e), token::NtPat(e) => self.pat_to_string(e), - &token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw.into()).to_string(), - token::NtLifetime(e) => e.to_string(), token::NtLiteral(e) => self.expr_to_string(e), token::NtVis(e) => self.vis_to_string(e), } @@ -915,10 +913,14 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere token::Literal(lit) => literal_to_string(lit).into(), /* Name components */ - token::Ident(s, is_raw) => { - IdentPrinter::new(s, is_raw.into(), convert_dollar_crate).to_string().into() + token::Ident(name, is_raw) => { + IdentPrinter::new(name, is_raw.into(), convert_dollar_crate).to_string().into() } - token::Lifetime(s) => s.to_string().into(), + token::NtIdent(ident, is_raw) => { + IdentPrinter::for_ast_ident(ident, is_raw.into()).to_string().into() + } + token::Lifetime(name) => name.to_string().into(), + token::NtLifetime(ident) => ident.name.to_string().into(), /* Other */ token::DocComment(comment_kind, attr_style, data) => { diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 897420a11cdfb..67a5a9128ccb6 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -201,10 +201,17 @@ impl<'a> StripUnconfigured<'a> { inner = self.configure_tokens(&inner); Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter() } - AttrTokenTree::Token(ref token, _) - if let TokenKind::Interpolated(nt) = &token.kind => - { - panic!("Nonterminal should have been flattened at {:?}: {:?}", token.span, nt); + AttrTokenTree::Token( + Token { + kind: + TokenKind::NtIdent(..) + | TokenKind::NtLifetime(..) + | TokenKind::Interpolated(..), + .. + }, + _, + ) => { + panic!("Nonterminal should have been flattened: {:?}", tree); } AttrTokenTree::Token(token, spacing) => { Some(AttrTokenTree::Token(token, spacing)).into_iter() diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index 011aa95c8a116..e35eba0f859b8 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -261,6 +261,16 @@ pub(super) fn transcribe<'a>( // without wrapping them into groups. maybe_use_metavar_location(cx, &stack, sp, tt, &mut marker) } + MatchedSingle(ParseNtResult::Ident(ident, is_raw)) => { + marker.visit_span(&mut sp); + let kind = token::NtIdent(*ident, *is_raw); + TokenTree::token_alone(kind, sp) + } + MatchedSingle(ParseNtResult::Lifetime(ident)) => { + marker.visit_span(&mut sp); + let kind = token::NtLifetime(*ident); + TokenTree::token_alone(kind, sp) + } MatchedSingle(ParseNtResult::Nt(nt)) => { // Other variables are emitted into the output stream as groups with // `Delimiter::Invisible` to maintain parsing priorities. diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 8cf9658016182..1f3547c841a90 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -220,6 +220,12 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { trees.push(TokenTree::Ident(Ident { sym, is_raw: is_raw.into(), span })) } + NtIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident { + sym: ident.name, + is_raw: is_raw.into(), + span: ident.span, + })), + Lifetime(name) => { let ident = symbol::Ident::new(name, span).without_first_quote(); trees.extend([ @@ -227,6 +233,15 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { + let stream = TokenStream::token_alone(token::Lifetime(ident.name), ident.span); + trees.push(TokenTree::Group(Group { + delimiter: pm::Delimiter::None, + stream: Some(stream), + span: DelimSpan::from_single(span), + })) + } + Literal(token::Lit { kind, symbol, suffix }) => { trees.push(TokenTree::Literal(self::Literal { kind: FromInternal::from_internal(kind), @@ -259,14 +274,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { - trees.push(TokenTree::Ident(Ident { - sym: ident.name, - is_raw: matches!(is_raw, IdentIsRaw::Yes), - span: ident.span, - })) - } - Interpolated(nt) => { let stream = TokenStream::from_nonterminal_ast(&nt); // A hack used to pass AST fragments to attribute and derive diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 8e3cd0f8ea9d5..cbc7ce9ef7cfb 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -724,7 +724,9 @@ impl<'a> Parser<'a> { /// Returns the span of expr if it was not interpolated, or the span of the interpolated token. fn interpolated_or_expr_span(&self, expr: &Expr) -> Span { match self.prev_token.kind { - TokenKind::Interpolated(..) => self.prev_token.span, + TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => { + self.prev_token.span + } _ => expr.span, } } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index cc40c8a7126ed..3e0a98a55ae9b 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -407,6 +407,8 @@ pub(super) fn token_descr(token: &Token) -> String { (Some(TokenDescription::Keyword), _) => Some("keyword"), (Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"), (Some(TokenDescription::DocComment), _) => Some("doc comment"), + (None, TokenKind::NtIdent(..)) => Some("identifier"), + (None, TokenKind::NtLifetime(..)) => Some("lifetime"), (None, TokenKind::Interpolated(node)) => Some(node.descr()), (None, _) => None, }; @@ -1633,5 +1635,9 @@ pub enum FlatToken { #[derive(Clone, Debug)] pub enum ParseNtResult { Tt(TokenTree), + Ident(Ident, IdentIsRaw), + Lifetime(Ident), + + /// This case will eventually be removed, along with `Token::Interpolate`. Nt(Lrc), } diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 3a560382d327c..a6f0ab78b5c29 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -25,15 +25,13 @@ impl<'a> Parser<'a> { | NtPat(_) | NtExpr(_) | NtTy(_) - | NtIdent(..) | NtLiteral(_) // `true`, `false` | NtMeta(_) | NtPath(_) => true, NtItem(_) | NtBlock(_) - | NtVis(_) - | NtLifetime(_) => false, + | NtVis(_) => false, } } @@ -50,25 +48,30 @@ impl<'a> Parser<'a> { NonterminalKind::Literal => token.can_begin_literal_maybe_minus(), NonterminalKind::Vis => match token.kind { // The follow-set of :vis + "priv" keyword + interpolated - token::Comma | token::Ident(..) | token::Interpolated(_) => true, + token::Comma + | token::Ident(..) + | token::NtIdent(..) + | token::NtLifetime(..) + | token::Interpolated(_) => true, _ => token.can_begin_type(), }, NonterminalKind::Block => match &token.kind { token::OpenDelim(Delimiter::Brace) => true, + token::NtLifetime(..) => true, token::Interpolated(nt) => match &**nt { - NtBlock(_) | NtLifetime(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true, - NtItem(_) | NtPat(_) | NtTy(_) | NtIdent(..) | NtMeta(_) | NtPath(_) - | NtVis(_) => false, + NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true, + NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false, }, _ => false, }, NonterminalKind::Path | NonterminalKind::Meta => match &token.kind { - token::PathSep | token::Ident(..) => true, + token::PathSep | token::Ident(..) | token::NtIdent(..) => true, token::Interpolated(nt) => may_be_ident(nt), _ => false, }, NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind { - token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) + // box, ref, mut, and other identifiers (can stricten) + token::Ident(..) | token::NtIdent(..) | token::OpenDelim(Delimiter::Parenthesis) | // tuple pattern token::OpenDelim(Delimiter::Bracket) | // slice pattern token::BinOp(token::And) | // reference @@ -86,10 +89,7 @@ impl<'a> Parser<'a> { _ => false, }, NonterminalKind::Lifetime => match &token.kind { - token::Lifetime(_) => true, - token::Interpolated(nt) => { - matches!(&**nt, NtLifetime(_)) - } + token::Lifetime(_) | token::NtLifetime(..) => true, _ => false, }, NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => { @@ -154,15 +154,16 @@ impl<'a> Parser<'a> { } // this could be handled like a token, since it is one - NonterminalKind::Ident if let Some((ident, is_raw)) = get_macro_ident(&self.token) => { - self.bump(); - NtIdent(ident, is_raw) - } NonterminalKind::Ident => { - return Err(self.dcx().create_err(UnexpectedNonterminal::Ident { - span: self.token.span, - token: self.token.clone(), - })); + return if let Some((ident, is_raw)) = get_macro_ident(&self.token) { + self.bump(); + Ok(ParseNtResult::Ident(ident, is_raw)) + } else { + Err(self.dcx().create_err(UnexpectedNonterminal::Ident { + span: self.token.span, + token: self.token.clone(), + })) + }; } NonterminalKind::Path => { NtPath(P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?)) @@ -173,14 +174,14 @@ impl<'a> Parser<'a> { .collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?)) } NonterminalKind::Lifetime => { - if self.check_lifetime() { - NtLifetime(self.expect_lifetime().ident) + return if self.check_lifetime() { + Ok(ParseNtResult::Lifetime(self.expect_lifetime().ident)) } else { - return Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime { + Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime { span: self.token.span, token: self.token.clone(), - })); - } + })) + }; } };