diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 9eb934c0c9e74..7224b482ed780 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -2423,6 +2423,7 @@ pub struct Attribute { /// or the construct this attribute is contained within (inner). pub style: AttrStyle, pub span: Span, + pub tokens: Option, } #[derive(Clone, Encodable, Decodable, Debug)] diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 8351be222f6bd..34b03382c5231 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -325,7 +325,7 @@ pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attri } pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute { - Attribute { kind: AttrKind::Normal(item), id: mk_attr_id(), style, span } + Attribute { kind: AttrKind::Normal(item), id: mk_attr_id(), style, span, tokens: None } } /// Returns an inner attribute with the given value and span. @@ -344,7 +344,13 @@ pub fn mk_doc_comment( data: Symbol, span: Span, ) -> Attribute { - Attribute { kind: AttrKind::DocComment(comment_kind, data), id: mk_attr_id(), style, span } + Attribute { + kind: AttrKind::DocComment(comment_kind, data), + id: mk_attr_id(), + style, + span, + tokens: None, + } } pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool { diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 382003c834ec9..166d36ce42433 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -577,7 +577,7 @@ pub fn noop_visit_local(local: &mut P, vis: &mut T) { } pub fn noop_visit_attribute(attr: &mut Attribute, vis: &mut T) { - let Attribute { kind, id: _, style: _, span } = attr; + let Attribute { kind, id: _, style: _, span, tokens: _ } = attr; match kind { AttrKind::Normal(AttrItem { path, args, tokens: _ }) => { vis.visit_path(path); diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index c49fd76a313d6..a6ac056b93b5e 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -210,9 +210,9 @@ impl<'hir> LoweringContext<'_, 'hir> { ex.span = e.span; } // Merge attributes into the inner expression. - let mut attrs = e.attrs.clone(); + let mut attrs: Vec<_> = e.attrs.iter().map(|a| self.lower_attr(a)).collect(); attrs.extend::>(ex.attrs.into()); - ex.attrs = attrs; + ex.attrs = attrs.into(); return ex; } @@ -1471,13 +1471,15 @@ impl<'hir> LoweringContext<'_, 'hir> { hir::MatchSource::ForLoopDesugar, )); + let attrs: Vec<_> = e.attrs.iter().map(|a| self.lower_attr(a)).collect(); + // This is effectively `{ let _result = ...; _result }`. // The construct was introduced in #21984 and is necessary to make sure that // temporaries in the `head` expression are dropped and do not leak to the // surrounding scope of the `match` since the `match` is not a terminating scope. // // Also, add the attributes to the outer returned expr node. - self.expr_drop_temps_mut(desugared_span, match_expr, e.attrs.clone()) + self.expr_drop_temps_mut(desugared_span, match_expr, attrs.into()) } /// Desugar `ExprKind::Try` from: `?` into: diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index a28d022c66139..361bccd7a252c 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -972,7 +972,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data), }; - Attribute { kind, id: attr.id, style: attr.style, span: attr.span } + // Tokens aren't needed after macro expansion and parsing + Attribute { kind, id: attr.id, style: attr.style, span: attr.span, tokens: None } } fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs { @@ -1713,7 +1714,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { pat: self.lower_pat(&l.pat), init, span: l.span, - attrs: l.attrs.clone(), + attrs: l.attrs.iter().map(|a| self.lower_attr(a)).collect::>().into(), source: hir::LocalSource::Normal, }, ids, diff --git a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs index 5ed8b69d92ab7..747e48ece704c 100644 --- a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs +++ b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs @@ -15,7 +15,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) - ); let start_span = parser.token.span; - let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item() { + let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item(false) { Ok(ai) => ai, Err(mut err) => { err.emit(); diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index dd087ab91509b..3551b92967c47 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -3,10 +3,13 @@ use rustc_ast::attr::HasAttrs; use rustc_ast::mut_visit::*; use rustc_ast::ptr::P; +use rustc_ast::token::{DelimToken, Token, TokenKind}; +use rustc_ast::tokenstream::{DelimSpan, LazyTokenStreamInner, Spacing, TokenStream, TokenTree}; use rustc_ast::{self as ast, AttrItem, Attribute, MetaItem}; use rustc_attr as attr; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::map_in_place::MapInPlace; +use rustc_data_structures::sync::Lrc; use rustc_errors::{error_code, struct_span_err, Applicability, Handler}; use rustc_feature::{Feature, Features, State as FeatureState}; use rustc_feature::{ @@ -289,7 +292,37 @@ impl<'a> StripUnconfigured<'a> { expanded_attrs .into_iter() .flat_map(|(item, span)| { - let attr = attr::mk_attr_from_item(attr.style, item, span); + let orig_tokens = + attr.tokens.as_ref().unwrap_or_else(|| panic!("Missing tokens for {:?}", attr)); + + // We are taking an attribute of the form `#[cfg_attr(pred, attr)]` + // and producing an attribute of the form `#[attr]`. We + // have captured tokens for `attr` itself, but we need to + // synthesize tokens for the wrapper `#` and `[]`, which + // we do below. + + // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token + // for `attr` when we expand it to `#[attr]` + let pound_token = orig_tokens.into_token_stream().trees().next().unwrap(); + if !matches!(pound_token, TokenTree::Token(Token { kind: TokenKind::Pound, .. })) { + panic!("Bad tokens for attribute {:?}", attr); + } + // We don't really have a good span to use for the syntheized `[]` + // in `#[attr]`, so just use the span of the `#` token. + let bracket_group = TokenTree::Delimited( + DelimSpan::from_single(pound_token.span()), + DelimToken::Bracket, + item.tokens + .clone() + .unwrap_or_else(|| panic!("Missing tokens for {:?}", item)) + .into_token_stream(), + ); + + let mut attr = attr::mk_attr_from_item(attr.style, item, span); + attr.tokens = Some(Lrc::new(LazyTokenStreamInner::Ready(TokenStream::new(vec![ + (pound_token, Spacing::Alone), + (bracket_group, Spacing::Alone), + ])))); self.process_cfg_attr(attr) }) .collect() diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 0b43225a242f7..ce198b3a41c9e 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -1785,6 +1785,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { span: at.span, id: at.id, style: at.style, + tokens: None, }; } else { noop_visit_attribute(at, self) diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index d9c24cc399482..bbb47a6e8071e 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -2,8 +2,9 @@ use crate::interface::{Compiler, Result}; use crate::proc_macro_decls; use crate::util; -use rustc_ast::mut_visit::MutVisitor; -use rustc_ast::{self as ast, visit}; +use rustc_ast::mut_visit::{self, MutVisitor}; +use rustc_ast::ptr::P; +use rustc_ast::{self as ast, token, visit}; use rustc_codegen_ssa::back::link::emit_metadata; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_data_structures::sync::{par_iter, Lrc, OnceCell, ParallelIterator, WorkerLocal}; @@ -36,6 +37,7 @@ use rustc_span::symbol::Symbol; use rustc_span::{FileName, RealFileName}; use rustc_trait_selection::traits; use rustc_typeck as typeck; +use smallvec::SmallVec; use tracing::{info, warn}; use rustc_serialize::json; @@ -50,6 +52,64 @@ use std::path::PathBuf; use std::rc::Rc; use std::{env, fs, iter, mem}; +/// Remove alls `LazyTokenStreams` from an AST struct +/// Normally, this is done during AST lowering. However, +/// printing the AST JSON requires us to serialize +/// the entire AST, and we don't want to serialize +/// a `LazyTokenStream`. +struct TokenStripper; +impl mut_visit::MutVisitor for TokenStripper { + fn flat_map_item(&mut self, mut i: P) -> SmallVec<[P; 1]> { + i.tokens = None; + mut_visit::noop_flat_map_item(i, self) + } + fn visit_block(&mut self, b: &mut P) { + b.tokens = None; + mut_visit::noop_visit_block(b, self); + } + fn flat_map_stmt(&mut self, mut stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + stmt.tokens = None; + mut_visit::noop_flat_map_stmt(stmt, self) + } + fn visit_pat(&mut self, p: &mut P) { + p.tokens = None; + mut_visit::noop_visit_pat(p, self); + } + fn visit_ty(&mut self, ty: &mut P) { + ty.tokens = None; + mut_visit::noop_visit_ty(ty, self); + } + fn visit_attribute(&mut self, attr: &mut ast::Attribute) { + attr.tokens = None; + if let ast::AttrKind::Normal(ast::AttrItem { tokens, .. }) = &mut attr.kind { + *tokens = None; + } + mut_visit::noop_visit_attribute(attr, self); + } + + fn visit_interpolated(&mut self, nt: &mut token::Nonterminal) { + if let token::Nonterminal::NtMeta(meta) = nt { + meta.tokens = None; + } + // Handles all of the other cases + mut_visit::noop_visit_interpolated(nt, self); + } + + fn visit_path(&mut self, p: &mut ast::Path) { + p.tokens = None; + mut_visit::noop_visit_path(p, self); + } + fn visit_vis(&mut self, vis: &mut ast::Visibility) { + vis.tokens = None; + mut_visit::noop_visit_vis(vis, self); + } + fn visit_expr(&mut self, e: &mut P) { + e.tokens = None; + mut_visit::noop_visit_expr(e, self); + } + fn visit_mac(&mut self, _mac: &mut ast::MacCall) {} +} + pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> { let krate = sess.time("parse_crate", || match input { Input::File(file) => parse_crate_from_file(file, &sess.parse_sess), @@ -59,6 +119,10 @@ pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> { })?; if sess.opts.debugging_opts.ast_json_noexpand { + // Set any `token` fields to `None` before + // we display the AST. + let mut krate = krate.clone(); + TokenStripper.visit_crate(&mut krate); println!("{}", json::as_json(&krate)); } @@ -379,6 +443,10 @@ fn configure_and_expand_inner<'a>( } if sess.opts.debugging_opts.ast_json { + // Set any `token` fields to `None` before + // we display the AST. + let mut krate = krate.clone(); + TokenStripper.visit_crate(&mut krate); println!("{}", json::as_json(&krate)); } diff --git a/compiler/rustc_middle/src/ich/impls_syntax.rs b/compiler/rustc_middle/src/ich/impls_syntax.rs index 7aba4fc64a929..cab2ca2919f9f 100644 --- a/compiler/rustc_middle/src/ich/impls_syntax.rs +++ b/compiler/rustc_middle/src/ich/impls_syntax.rs @@ -40,11 +40,12 @@ impl<'ctx> rustc_ast::HashStableContext for StableHashingContext<'ctx> { debug_assert!(!attr.ident().map_or(false, |ident| self.is_ignored_attr(ident.name))); debug_assert!(!attr.is_doc_comment()); - let ast::Attribute { kind, id: _, style, span } = attr; + let ast::Attribute { kind, id: _, style, span, tokens } = attr; if let ast::AttrKind::Normal(item) = kind { item.hash_stable(self, hasher); style.hash_stable(self, hasher); span.hash_stable(self, hasher); + tokens.as_ref().expect_none("Tokens should have been removed during lowering!"); } else { unreachable!(); } diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index e073f57108838..ba416be6b38d3 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -252,9 +252,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke let convert_tokens = |tokens: Option| tokens.map(|t| t.into_token_stream()); let tokens = match *nt { - Nonterminal::NtItem(ref item) => { - prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) - } + Nonterminal::NtItem(ref item) => prepend_attrs(&item.attrs, item.tokens.as_ref()), Nonterminal::NtBlock(ref block) => convert_tokens(block.tokens.clone()), Nonterminal::NtStmt(ref stmt) => { // FIXME: We currently only collect tokens for `:stmt` @@ -279,7 +277,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke if expr.tokens.is_none() { debug!("missing tokens for expr {:?}", expr); } - prepend_attrs(sess, &expr.attrs, expr.tokens.as_ref(), span) + prepend_attrs(&expr.attrs, expr.tokens.as_ref()) } }; @@ -603,10 +601,8 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool { } fn prepend_attrs( - sess: &ParseSess, attrs: &[ast::Attribute], tokens: Option<&tokenstream::LazyTokenStream>, - span: rustc_span::Span, ) -> Option { let tokens = tokens?.clone().into_token_stream(); if attrs.is_empty() { @@ -619,47 +615,12 @@ fn prepend_attrs( ast::AttrStyle::Outer, "inner attributes should prevent cached tokens from existing" ); - - let source = pprust::attribute_to_string(attr); - let macro_filename = FileName::macro_expansion_source_code(&source); - - let item = match attr.kind { - ast::AttrKind::Normal(ref item) => item, - ast::AttrKind::DocComment(..) => { - let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); - builder.push(stream); - continue; - } - }; - - // synthesize # [ $path $tokens ] manually here - let mut brackets = tokenstream::TokenStreamBuilder::new(); - - // For simple paths, push the identifier directly - if item.path.segments.len() == 1 && item.path.segments[0].args.is_none() { - let ident = item.path.segments[0].ident; - let token = token::Ident(ident.name, ident.as_str().starts_with("r#")); - brackets.push(tokenstream::TokenTree::token(token, ident.span)); - - // ... and for more complicated paths, fall back to a reparse hack that - // should eventually be removed. - } else { - let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); - brackets.push(stream); - } - - brackets.push(item.args.outer_tokens()); - - // The span we list here for `#` and for `[ ... ]` are both wrong in - // that it encompasses more than each token, but it hopefully is "good - // enough" for now at least. - builder.push(tokenstream::TokenTree::token(token::Pound, attr.span)); - let delim_span = tokenstream::DelimSpan::from_single(attr.span); - builder.push(tokenstream::TokenTree::Delimited( - delim_span, - token::DelimToken::Bracket, - brackets.build(), - )); + builder.push( + attr.tokens + .clone() + .unwrap_or_else(|| panic!("Attribute {:?} is missing tokens!", attr)) + .into_token_stream(), + ); } builder.push(tokens.clone()); Some(builder.build()) diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 73439643d69b9..053b7e0b75fe4 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -30,41 +30,53 @@ impl<'a> Parser<'a> { let mut just_parsed_doc_comment = false; loop { debug!("parse_outer_attributes: self.token={:?}", self.token); - if self.check(&token::Pound) { - let inner_error_reason = if just_parsed_doc_comment { - "an inner attribute is not permitted following an outer doc comment" - } else if !attrs.is_empty() { - "an inner attribute is not permitted following an outer attribute" - } else { - DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG - }; - let inner_parse_policy = InnerAttrPolicy::Forbidden { - reason: inner_error_reason, - saw_doc_comment: just_parsed_doc_comment, - prev_attr_sp: attrs.last().map(|a| a.span), - }; - let attr = self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?; - attrs.push(attr); - just_parsed_doc_comment = false; + let (attr, tokens) = if self.check(&token::Pound) { + self.collect_tokens(|this| { + let inner_error_reason = if just_parsed_doc_comment { + "an inner attribute is not permitted following an outer doc comment" + } else if !attrs.is_empty() { + "an inner attribute is not permitted following an outer attribute" + } else { + DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG + }; + let inner_parse_policy = InnerAttrPolicy::Forbidden { + reason: inner_error_reason, + saw_doc_comment: just_parsed_doc_comment, + prev_attr_sp: attrs.last().map(|a| a.span), + }; + let attr = this.parse_attribute_with_inner_parse_policy(inner_parse_policy)?; + just_parsed_doc_comment = false; + Ok(Some(attr)) + })? } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind { - let attr = attr::mk_doc_comment(comment_kind, attr_style, data, self.token.span); - if attr.style != ast::AttrStyle::Outer { - self.sess - .span_diagnostic - .struct_span_err_with_code( - self.token.span, - "expected outer doc comment", - error_code!(E0753), - ) - .note( - "inner doc comments like this (starting with \ - `//!` or `/*!`) can only appear before items", - ) - .emit(); - } + self.collect_tokens(|this| { + let attr = + attr::mk_doc_comment(comment_kind, attr_style, data, this.token.span); + if attr.style != ast::AttrStyle::Outer { + this.sess + .span_diagnostic + .struct_span_err_with_code( + this.token.span, + "expected outer doc comment", + error_code!(E0753), + ) + .note( + "inner doc comments like this (starting with \ + `//!` or `/*!`) can only appear before items", + ) + .emit(); + } + this.bump(); + just_parsed_doc_comment = true; + Ok(Some(attr)) + })? + } else { + (None, None) + }; + + if let Some(mut attr) = attr { + attr.tokens = tokens; attrs.push(attr); - self.bump(); - just_parsed_doc_comment = true; } else { break; } @@ -99,7 +111,7 @@ impl<'a> Parser<'a> { if self.eat(&token::Not) { ast::AttrStyle::Inner } else { ast::AttrStyle::Outer }; self.expect(&token::OpenDelim(token::Bracket))?; - let item = self.parse_attr_item()?; + let item = self.parse_attr_item(false)?; self.expect(&token::CloseDelim(token::Bracket))?; let attr_sp = lo.to(self.prev_token.span); @@ -148,7 +160,7 @@ impl<'a> Parser<'a> { /// PATH /// PATH `=` UNSUFFIXED_LIT /// The delimiters or `=` are still put into the resulting token stream. - pub fn parse_attr_item(&mut self) -> PResult<'a, ast::AttrItem> { + pub fn parse_attr_item(&mut self, capture_tokens: bool) -> PResult<'a, ast::AttrItem> { let item = match self.token.kind { token::Interpolated(ref nt) => match **nt { Nonterminal::NtMeta(ref item) => Some(item.clone().into_inner()), @@ -160,9 +172,18 @@ impl<'a> Parser<'a> { self.bump(); item } else { - let path = self.parse_path(PathStyle::Mod)?; - let args = self.parse_attr_args()?; - ast::AttrItem { path, args, tokens: None } + let do_parse = |this: &mut Self| { + let path = this.parse_path(PathStyle::Mod)?; + let args = this.parse_attr_args()?; + Ok(ast::AttrItem { path, args, tokens: None }) + }; + if capture_tokens { + let (mut item, tokens) = self.collect_tokens(do_parse)?; + item.tokens = tokens; + item + } else { + do_parse(self)? + } }) } @@ -175,19 +196,31 @@ impl<'a> Parser<'a> { let mut attrs: Vec = vec![]; loop { // Only try to parse if it is an inner attribute (has `!`). - if self.check(&token::Pound) && self.look_ahead(1, |t| t == &token::Not) { - let attr = self.parse_attribute(true)?; - assert_eq!(attr.style, ast::AttrStyle::Inner); - attrs.push(attr); - } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind { - // We need to get the position of this token before we bump. - let attr = attr::mk_doc_comment(comment_kind, attr_style, data, self.token.span); - if attr.style == ast::AttrStyle::Inner { - attrs.push(attr); - self.bump(); + let (attr, tokens) = + if self.check(&token::Pound) && self.look_ahead(1, |t| t == &token::Not) { + self.collect_tokens(|this| { + let attr = this.parse_attribute(true)?; + assert_eq!(attr.style, ast::AttrStyle::Inner); + Ok(Some(attr)) + })? + } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind { + self.collect_tokens(|this| { + // We need to get the position of this token before we bump. + let attr = + attr::mk_doc_comment(comment_kind, attr_style, data, this.token.span); + if attr.style == ast::AttrStyle::Inner { + this.bump(); + Ok(Some(attr)) + } else { + Ok(None) + } + })? } else { - break; - } + (None, None) + }; + if let Some(mut attr) = attr { + attr.tokens = tokens; + attrs.push(attr); } else { break; } @@ -220,7 +253,7 @@ impl<'a> Parser<'a> { let mut expanded_attrs = Vec::with_capacity(1); while self.token.kind != token::Eof { let lo = self.token.span; - let item = self.parse_attr_item()?; + let item = self.parse_attr_item(true)?; expanded_attrs.push((item, lo.to(self.prev_token.span))); if !self.eat(&token::Comma) { break; diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 698a7e7d9cde8..a026353d82525 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -1116,7 +1116,7 @@ impl<'a> Parser<'a> { ) -> PResult<'a, P> { if needs_tokens { let (mut expr, tokens) = self.collect_tokens(f)?; - expr.tokens = Some(tokens); + expr.tokens = tokens; Ok(expr) } else { f(self) diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 4ad259715bd98..e57a2e42b5dde 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -151,7 +151,7 @@ impl<'a> Parser<'a> { if let Some(tokens) = tokens { if let Some(item) = &mut item { if !item.attrs.iter().any(|attr| attr.style == AttrStyle::Inner) { - item.tokens = Some(tokens); + item.tokens = tokens; } } } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index fb825256d92cd..8ab39b6e6d8fa 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -1178,8 +1178,9 @@ impl<'a> Parser<'a> { /// Records all tokens consumed by the provided callback, /// including the current token. These tokens are collected - /// into a `TokenStream`, and returned along with the result - /// of the callback. + /// into a `LazyTokenStream`, and returned along with the result + /// of the callback. The returned `LazyTokenStream` will be `None` + /// if not tokens were captured. /// /// Note: If your callback consumes an opening delimiter /// (including the case where you call `collect_tokens` @@ -1195,7 +1196,7 @@ impl<'a> Parser<'a> { pub fn collect_tokens( &mut self, f: impl FnOnce(&mut Self) -> PResult<'a, R>, - ) -> PResult<'a, (R, LazyTokenStream)> { + ) -> PResult<'a, (R, Option)> { let start_token = (self.token.clone(), self.token_spacing); let mut cursor_snapshot = self.token_cursor.clone(); @@ -1205,6 +1206,11 @@ impl<'a> Parser<'a> { let num_calls = new_calls - cursor_snapshot.num_next_calls; let desugar_doc_comments = self.desugar_doc_comments; + // We didn't capture any tokens + if num_calls == 0 { + return Ok((ret, None)); + } + // Produces a `TokenStream` on-demand. Using `cursor_snapshot` // and `num_calls`, we can reconstruct the `TokenStream` seen // by the callback. This allows us to avoid producing a `TokenStream` @@ -1233,7 +1239,7 @@ impl<'a> Parser<'a> { }; let stream = LazyTokenStream::new(LazyTokenStreamInner::Lazy(Box::new(lazy_cb))); - Ok((ret, stream)) + Ok((ret, Some(stream))) } /// `::{` or `::*` diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 15660fd574c13..98fb1c8292510 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -103,7 +103,7 @@ impl<'a> Parser<'a> { // If we captured tokens during parsing (due to outer attributes), // use those. if item.tokens.is_none() { - item.tokens = Some(tokens); + item.tokens = tokens; } token::NtItem(item) } @@ -115,7 +115,7 @@ impl<'a> Parser<'a> { let (mut block, tokens) = self.collect_tokens(|this| this.parse_block())?; // We have have eaten an NtBlock, which could already have tokens if block.tokens.is_none() { - block.tokens = Some(tokens); + block.tokens = tokens; } token::NtBlock(block) } @@ -124,7 +124,7 @@ impl<'a> Parser<'a> { match stmt { Some(mut s) => { if s.tokens.is_none() { - s.tokens = Some(tokens); + s.tokens = tokens; } token::NtStmt(s) } @@ -137,7 +137,7 @@ impl<'a> Parser<'a> { let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?; // We have have eaten an NtPat, which could already have tokens if pat.tokens.is_none() { - pat.tokens = Some(tokens); + pat.tokens = tokens; } token::NtPat(pat) } @@ -146,7 +146,7 @@ impl<'a> Parser<'a> { // If we captured tokens during parsing (due to outer attributes), // use those. if expr.tokens.is_none() { - expr.tokens = Some(tokens); + expr.tokens = tokens; } token::NtExpr(expr) } @@ -155,7 +155,7 @@ impl<'a> Parser<'a> { self.collect_tokens(|this| this.parse_literal_maybe_minus())?; // We have have eaten a nonterminal, which could already have tokens if lit.tokens.is_none() { - lit.tokens = Some(tokens); + lit.tokens = tokens; } token::NtLiteral(lit) } @@ -163,7 +163,7 @@ impl<'a> Parser<'a> { let (mut ty, tokens) = self.collect_tokens(|this| this.parse_ty())?; // We have an eaten an NtTy, which could already have tokens if ty.tokens.is_none() { - ty.tokens = Some(tokens); + ty.tokens = tokens; } token::NtTy(ty) } @@ -183,15 +183,15 @@ impl<'a> Parser<'a> { self.collect_tokens(|this| this.parse_path(PathStyle::Type))?; // We have have eaten an NtPath, which could already have tokens if path.tokens.is_none() { - path.tokens = Some(tokens); + path.tokens = tokens; } token::NtPath(path) } NonterminalKind::Meta => { - let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item())?; + let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item(false))?; // We may have eaten a nonterminal, which could already have tokens if attr.tokens.is_none() { - attr.tokens = Some(tokens); + attr.tokens = tokens; } token::NtMeta(P(attr)) } @@ -201,7 +201,7 @@ impl<'a> Parser<'a> { self.collect_tokens(|this| this.parse_visibility(FollowedByType::Yes))?; // We may have etan an `NtVis`, which could already have tokens if vis.tokens.is_none() { - vis.tokens = Some(tokens); + vis.tokens = tokens; } token::NtVis(vis) } diff --git a/src/test/ui/ast-json/ast-json-noexpand-output.stdout b/src/test/ui/ast-json/ast-json-noexpand-output.stdout index 3d7d476cf6c05..259206ba90718 100644 --- a/src/test/ui/ast-json/ast-json-noexpand-output.stdout +++ b/src/test/ui/ast-json/ast-json-noexpand-output.stdout @@ -1 +1 @@ -{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]} +{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":null}],"span":{"lo":0,"hi":0},"proc_macros":[]} diff --git a/src/test/ui/ast-json/ast-json-output.stdout b/src/test/ui/ast-json/ast-json-output.stdout index a4ce6061b4c9d..76e32044add18 100644 --- a/src/test/ui/ast-json/ast-json-output.stdout +++ b/src/test/ui/ast-json/ast-json-output.stdout @@ -1 +1 @@ -{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]} +{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":null}],"span":{"lo":0,"hi":0},"proc_macros":[]} diff --git a/src/test/ui/proc-macro/issue-75930-derive-cfg.stdout b/src/test/ui/proc-macro/issue-75930-derive-cfg.stdout index f3daa56a49c45..4c0810217bf33 100644 --- a/src/test/ui/proc-macro/issue-75930-derive-cfg.stdout +++ b/src/test/ui/proc-macro/issue-75930-derive-cfg.stdout @@ -26,7 +26,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [ Punct { ch: '#', spacing: Alone, - span: $DIR/issue-75930-derive-cfg.rs:17:24: 17:40 (#0), + span: $DIR/issue-75930-derive-cfg.rs:17:1: 17:2 (#0), }, Group { delimiter: Bracket, @@ -46,12 +46,12 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [ span: $DIR/issue-75930-derive-cfg.rs:17:29: 17:40 (#0), }, ], - span: $DIR/issue-75930-derive-cfg.rs:17:24: 17:40 (#0), + span: $DIR/issue-75930-derive-cfg.rs:17:1: 17:2 (#0), }, Punct { ch: '#', spacing: Alone, - span: $DIR/issue-75930-derive-cfg.rs:19:1: 19:17 (#0), + span: $DIR/issue-75930-derive-cfg.rs:19:1: 19:2 (#0), }, Group { delimiter: Bracket, @@ -71,12 +71,12 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [ span: $DIR/issue-75930-derive-cfg.rs:19:9: 19:16 (#0), }, ], - span: $DIR/issue-75930-derive-cfg.rs:19:1: 19:17 (#0), + span: $DIR/issue-75930-derive-cfg.rs:19:2: 19:17 (#0), }, Punct { ch: '#', spacing: Alone, - span: $DIR/issue-75930-derive-cfg.rs:20:1: 20:19 (#0), + span: $DIR/issue-75930-derive-cfg.rs:20:1: 20:2 (#0), }, Group { delimiter: Bracket, @@ -96,12 +96,12 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [ span: $DIR/issue-75930-derive-cfg.rs:20:15: 20:18 (#0), }, ], - span: $DIR/issue-75930-derive-cfg.rs:20:1: 20:19 (#0), + span: $DIR/issue-75930-derive-cfg.rs:20:2: 20:19 (#0), }, Punct { ch: '#', spacing: Alone, - span: $DIR/issue-75930-derive-cfg.rs:16:1: 16:19 (#0), + span: $DIR/issue-75930-derive-cfg.rs:16:1: 16:2 (#0), }, Group { delimiter: Bracket, @@ -121,7 +121,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [ span: $DIR/issue-75930-derive-cfg.rs:16:15: 16:18 (#0), }, ], - span: $DIR/issue-75930-derive-cfg.rs:16:1: 16:19 (#0), + span: $DIR/issue-75930-derive-cfg.rs:16:2: 16:19 (#0), }, Ident { ident: "struct",