From 6b8bf206d73e64eacd360739f92f4f65f37f5992 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Mon, 21 Sep 2020 17:13:32 -0400 Subject: [PATCH] Only collect tokens for attributes themselves when needed We need attribute tokens if the target supports custom attributes, or we are inside a `derive` target. Otherwise, we don't need tokens for individual tokens - any needed token collection will be handled further up the stack. --- compiler/rustc_parse/src/parser/attr.rs | 53 +++++++++--- compiler/rustc_parse/src/parser/expr.rs | 73 +++++++++-------- compiler/rustc_parse/src/parser/generics.rs | 4 +- compiler/rustc_parse/src/parser/item.rs | 19 ++--- compiler/rustc_parse/src/parser/mod.rs | 4 +- compiler/rustc_parse/src/parser/pat.rs | 4 +- compiler/rustc_parse/src/parser/stmt.rs | 89 +++++++++++---------- 7 files changed, 141 insertions(+), 105 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 13ccb82354ca1..d2bf42dcdedb1 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -22,6 +22,12 @@ pub(super) enum InnerAttrPolicy<'a> { const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &str = "an inner attribute is not \ permitted in this context"; +#[derive(Copy, Clone, Eq, PartialEq)] +pub enum SupportsCustomAttr { + Yes, + No, +} + pub struct CfgAttrItem { pub item: ast::AttrItem, pub span: Span, @@ -39,12 +45,15 @@ impl<'a> Parser<'a> { self.check(&token::Pound) || matches!(self.token.kind, token::DocComment(..)) } - fn parse_outer_attributes_(&mut self) -> PResult<'a, Vec> { + fn parse_outer_attributes_( + &mut self, + custom: SupportsCustomAttr, + ) -> PResult<'a, Vec> { let mut attrs: Vec = Vec::new(); let mut just_parsed_doc_comment = false; loop { - let (attr, tokens) = self.collect_tokens_keep_in_stream(false, |this| { + let mut parse_attr = |this: &mut Self| { debug!("parse_outer_attributes: self.token={:?}", this.token); if this.check(&token::Pound) { let inner_error_reason = if just_parsed_doc_comment { @@ -85,9 +94,21 @@ impl<'a> Parser<'a> { } else { Ok((None, Vec::new())) } - })?; + }; + + // `in_derive` does not take into account the attributes we are currently parsing + // (which may contain a `derive`). This is fine - if a `derive` attribute + // can legally occur here, `custom` will be `SupportsCustomAttr::Yes` + let (attr, tokens) = if custom == SupportsCustomAttr::Yes || self.in_derive { + let (attr, tokens) = self.collect_tokens_keep_in_stream(false, parse_attr)?; + (attr, Some(tokens)) + } else { + let (attr, _nested_attrs) = parse_attr(self)?; + (attr, None) + }; + if let Some(mut attr) = attr { - attr.tokens = Some(tokens.to_tokenstream()); + attr.tokens = tokens.map(|t| t.to_tokenstream()); attrs.push(attr); } else { break; @@ -102,6 +123,7 @@ impl<'a> Parser<'a> { >( &mut self, already_parsed_attrs: Option, + custom: SupportsCustomAttr, f: F, ) -> PResult<'a, (R, Option)> { let in_derive = self.in_derive; @@ -119,10 +141,17 @@ impl<'a> Parser<'a> { let mut res = res?; - res.visit_attrs(|attrs| { - new_attrs = attrs.clone(); - }); - Ok((res, new_attrs)) + // `this.in_derive` does not take into account our new attributes + // (which may contain a `derive`). This is fine - if a `derive` attribute + // can legally occur here, `custom` will be `SupportsCustomAttr::Yes` + if custom == SupportsCustomAttr::Yes || this.in_derive { + res.visit_attrs(|attrs| { + new_attrs = attrs.clone(); + }); + Ok((res, new_attrs)) + } else { + Ok((res, Vec::new())) + } })?; Ok((res, Some(tokens))) }; @@ -142,7 +171,7 @@ impl<'a> Parser<'a> { return Ok((f(self, AttrVec::new())?, None)); } - let attrs = self.parse_outer_attributes_()?; + let attrs = self.parse_outer_attributes_(custom)?; if !needs_tokens(&attrs) { return Ok((f(self, attrs.into())?, None)); } @@ -153,17 +182,19 @@ impl<'a> Parser<'a> { pub(super) fn parse_outer_attributes( &mut self, + custom: SupportsCustomAttr, f: impl FnOnce(&mut Self, Vec) -> PResult<'a, R>, ) -> PResult<'a, R> { - self.parse_outer_attributes_with_tokens(f).map(|(res, _tokens)| res) + self.parse_outer_attributes_with_tokens(custom, f).map(|(res, _tokens)| res) } /// Parses attributes that appear before an item. pub(super) fn parse_outer_attributes_with_tokens( &mut self, + custom: SupportsCustomAttr, f: impl FnOnce(&mut Self, Vec) -> PResult<'a, R>, ) -> PResult<'a, (R, Option)> { - self.parse_or_use_outer_attributes(None, |this, attrs| f(this, attrs.into())) + self.parse_or_use_outer_attributes(None, custom, |this, attrs| f(this, attrs.into())) } /// Matches `attribute = # ! [ meta_item ]`. diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 762d1c1c82b4a..a0e089135c5b7 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -1,7 +1,7 @@ use super::pat::{GateOr, PARAM_EXPECTED}; use super::ty::{AllowPlus, RecoverQPath}; use super::{BlockMode, Parser, PathStyle, Restrictions, TokenType}; -use super::{SemiColonMode, SeqSep, TokenExpectType}; +use super::{SemiColonMode, SeqSep, SupportsCustomAttr, TokenExpectType}; use crate::maybe_recover_from_interpolated_ty_qpath; use rustc_ast::ptr::P; @@ -445,41 +445,43 @@ impl<'a> Parser<'a> { _ => RangeLimits::Closed, }; let op = AssocOp::from_token(&self.token); - let (mut expr, tokens) = self.parse_or_use_outer_attributes(attrs, |this, attrs| { - let lo = this.token.span; - this.bump(); - let (span, opt_end) = if this.is_at_start_of_range_notation_rhs() { - // RHS must be parsed with more associativity than the dots. - this.parse_assoc_expr_with(op.unwrap().precedence() + 1, LhsExpr::NotYetParsed) - .map(|x| (lo.to(x.span), Some(x)))? - } else { - (lo, None) - }; - Ok(this.mk_expr(span, this.mk_range(None, opt_end, limits)?, attrs)) - })?; + let (mut expr, tokens) = + self.parse_or_use_outer_attributes(attrs, SupportsCustomAttr::Yes, |this, attrs| { + let lo = this.token.span; + this.bump(); + let (span, opt_end) = if this.is_at_start_of_range_notation_rhs() { + // RHS must be parsed with more associativity than the dots. + this.parse_assoc_expr_with(op.unwrap().precedence() + 1, LhsExpr::NotYetParsed) + .map(|x| (lo.to(x.span), Some(x)))? + } else { + (lo, None) + }; + Ok(this.mk_expr(span, this.mk_range(None, opt_end, limits)?, attrs)) + })?; expr.tokens = tokens; Ok(expr) } /// Parses a prefix-unary-operator expr. fn parse_prefix_expr(&mut self, attrs: Option) -> PResult<'a, P> { - let (mut expr, tokens) = self.parse_or_use_outer_attributes(attrs, |this, attrs| { - let lo = this.token.span; - // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr() - let (hi, ex) = match this.token.uninterpolate().kind { - token::Not => this.parse_unary_expr(lo, UnOp::Not), // `!expr` - token::Tilde => this.recover_tilde_expr(lo), // `~expr` - token::BinOp(token::Minus) => this.parse_unary_expr(lo, UnOp::Neg), // `-expr` - token::BinOp(token::Star) => this.parse_unary_expr(lo, UnOp::Deref), // `*expr` - token::BinOp(token::And) | token::AndAnd => this.parse_borrow_expr(lo), - token::Ident(..) if this.token.is_keyword(kw::Box) => this.parse_box_expr(lo), - token::Ident(..) if this.is_mistaken_not_ident_negation() => { - this.recover_not_expr(lo) - } - _ => return this.parse_dot_or_call_expr(attrs), - }?; - Ok(this.mk_expr(lo.to(hi), ex, attrs)) - })?; + let (mut expr, tokens) = + self.parse_or_use_outer_attributes(attrs, SupportsCustomAttr::Yes, |this, attrs| { + let lo = this.token.span; + // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr() + let (hi, ex) = match this.token.uninterpolate().kind { + token::Not => this.parse_unary_expr(lo, UnOp::Not), // `!expr` + token::Tilde => this.recover_tilde_expr(lo), // `~expr` + token::BinOp(token::Minus) => this.parse_unary_expr(lo, UnOp::Neg), // `-expr` + token::BinOp(token::Star) => this.parse_unary_expr(lo, UnOp::Deref), // `*expr` + token::BinOp(token::And) | token::AndAnd => this.parse_borrow_expr(lo), + token::Ident(..) if this.token.is_keyword(kw::Box) => this.parse_box_expr(lo), + token::Ident(..) if this.is_mistaken_not_ident_negation() => { + this.recover_not_expr(lo) + } + _ => return this.parse_dot_or_call_expr(attrs), + }?; + Ok(this.mk_expr(lo.to(hi), ex, attrs)) + })?; expr.tokens = tokens; Ok(expr) } @@ -1598,7 +1600,7 @@ impl<'a> Parser<'a> { /// Parses a parameter in a closure header (e.g., `|arg, arg|`). fn parse_fn_block_param(&mut self) -> PResult<'a, Param> { let lo = self.token.span; - self.parse_outer_attributes(|this, attrs| { + self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| { let pat = this.parse_pat(PARAM_EXPECTED)?; let ty = if this.eat(&token::Colon) { this.parse_ty()? @@ -1629,7 +1631,8 @@ impl<'a> Parser<'a> { self.error_missing_if_cond(lo, cond.span) } else { // For recovery. - let attrs = self.parse_outer_attributes(|_this, attrs| Ok(attrs))?; + let attrs = + self.parse_outer_attributes(SupportsCustomAttr::No, |_this, attrs| Ok(attrs))?; let not_block = self.token != token::OpenDelim(token::Brace); let block = self.parse_block().map_err(|mut err| { if not_block { @@ -1686,7 +1689,7 @@ impl<'a> Parser<'a> { /// Parses an `else { ... }` expression (`else` token already eaten). fn parse_else_expr(&mut self) -> PResult<'a, P> { let ctx_span = self.prev_token.span; // `else` - self.parse_outer_attributes(|this, attrs| { + self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| { let expr = if this.eat_keyword(kw::If) { this.parse_if_expr(AttrVec::new())? } else { @@ -1845,7 +1848,7 @@ impl<'a> Parser<'a> { } pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> { - self.parse_outer_attributes(|this, attrs| { + self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| { let lo = this.token.span; let pat = this.parse_top_pat(GateOr::No)?; let guard = if this.eat_keyword(kw::If) { @@ -2161,7 +2164,7 @@ impl<'a> Parser<'a> { /// Parses `ident (COLON expr)?`. fn parse_field(&mut self) -> PResult<'a, Field> { - self.parse_outer_attributes(|this, attrs| { + self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| { let attrs = attrs.into(); let lo = this.token.span; diff --git a/compiler/rustc_parse/src/parser/generics.rs b/compiler/rustc_parse/src/parser/generics.rs index b1421a83e7ee7..5a8664aa5222d 100644 --- a/compiler/rustc_parse/src/parser/generics.rs +++ b/compiler/rustc_parse/src/parser/generics.rs @@ -1,4 +1,4 @@ -use super::Parser; +use super::{Parser, SupportsCustomAttr}; use rustc_ast::token; use rustc_ast::{ @@ -74,7 +74,7 @@ impl<'a> Parser<'a> { let mut params = Vec::new(); loop { let mut should_break = false; - let param = self.parse_outer_attributes(|this, attrs| { + let param = self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| { let param = if this.check_lifetime() { let lifetime = this.expect_lifetime(); // Parse lifetime parameter. diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 4ae620c2ba3d3..8be1f7970d90c 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -1,6 +1,6 @@ use super::diagnostics::{dummy_arg, ConsumeClosingDelim, Error}; use super::ty::{AllowPlus, RecoverQPath}; -use super::{FollowedByType, Parser, PathStyle}; +use super::{FollowedByType, Parser, PathStyle, SupportsCustomAttr}; use crate::maybe_whole; use crate::parser::attr::attrs_require_tokens; @@ -101,9 +101,10 @@ impl<'a> Parser<'a> { } fn parse_item_(&mut self, req_name: ReqName) -> PResult<'a, Option> { - let res = self.parse_outer_attributes_with_tokens(|this, attrs| { - this.parse_item_common(attrs, true, false, req_name) - }); + let res = self + .parse_outer_attributes_with_tokens(SupportsCustomAttr::Yes, |this, attrs| { + this.parse_item_common(attrs, true, false, req_name) + }); res.map(|(mut item, tokens)| { if let Some(item) = item.as_mut() { if item.tokens.is_none() { @@ -1073,7 +1074,7 @@ impl<'a> Parser<'a> { } fn parse_enum_variant(&mut self) -> PResult<'a, Option> { - self.parse_outer_attributes(|this, variant_attrs| { + self.parse_outer_attributes(SupportsCustomAttr::No, |this, variant_attrs| { let vlo = this.token.span; let vis = this.parse_visibility(FollowedByType::No)?; @@ -1259,7 +1260,7 @@ impl<'a> Parser<'a> { // This is the case where we find `struct Foo(T) where T: Copy;` // Unit like structs are handled in parse_item_struct function self.parse_paren_comma_seq(|p| { - p.parse_outer_attributes(|p, attrs| { + p.parse_outer_attributes(SupportsCustomAttr::No, |p, attrs| { let lo = p.token.span; let vis = p.parse_visibility(FollowedByType::Yes)?; let ty = p.parse_ty()?; @@ -1279,7 +1280,7 @@ impl<'a> Parser<'a> { /// Parses an element of a struct declaration. fn parse_struct_decl_field(&mut self) -> PResult<'a, StructField> { - self.parse_outer_attributes(|this, attrs| { + self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| { let lo = this.token.span; let vis = this.parse_visibility(FollowedByType::No)?; this.parse_single_struct_field(lo, vis, attrs) @@ -1720,7 +1721,7 @@ impl<'a> Parser<'a> { /// - `self` is syntactically allowed when `first_param` holds. fn parse_param_general(&mut self, req_name: ReqName, first_param: bool) -> PResult<'a, Param> { let lo = self.token.span; - self.parse_outer_attributes(|this, attrs| { + self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| { // Possibly parse `self`. Recover if we parsed it and it wasn't allowed here. if let Some(mut param) = this.parse_self_param()? { param.attrs = attrs.into(); @@ -1908,7 +1909,7 @@ impl<'a> Parser<'a> { fn recover_first_param(&mut self) -> &'static str { let res = self - .parse_outer_attributes(|this, _attrs| this.parse_self_param()) + .parse_outer_attributes(SupportsCustomAttr::No, |this, _attrs| this.parse_self_param()) .map_err(|mut err| err.cancel()); match res { Ok(Some(_)) => "method", diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 2dbbff66b5480..99d46d795c91a 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -10,6 +10,7 @@ mod stmt; mod ty; use crate::lexer::UnmatchedBrace; +pub use attr::SupportsCustomAttr; use diagnostics::Error; pub use path::PathStyle; @@ -27,9 +28,8 @@ use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError use rustc_session::parse::ParseSess; use rustc_span::source_map::{Span, DUMMY_SP}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; -use tracing::debug; - use std::{cmp, mem, slice}; +use tracing::debug; bitflags::bitflags! { struct Restrictions: u8 { diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs index 88ff2547d5490..ac2ed4de0f341 100644 --- a/compiler/rustc_parse/src/parser/pat.rs +++ b/compiler/rustc_parse/src/parser/pat.rs @@ -1,4 +1,4 @@ -use super::{Parser, PathStyle}; +use super::{Parser, PathStyle, SupportsCustomAttr}; use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole}; use rustc_ast::mut_visit::{noop_visit_mac, noop_visit_pat, MutVisitor}; use rustc_ast::ptr::P; @@ -831,7 +831,7 @@ impl<'a> Parser<'a> { let mut etc_span = None; while self.token != token::CloseDelim(token::Brace) { - let field_pat = self.parse_outer_attributes(|this, attrs| { + let field_pat = self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| { let lo = this.token.span; // check that a comma comes after every field diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs index 324369b9304d1..03c6948f762ff 100644 --- a/compiler/rustc_parse/src/parser/stmt.rs +++ b/compiler/rustc_parse/src/parser/stmt.rs @@ -3,7 +3,7 @@ use super::diagnostics::Error; use super::expr::LhsExpr; use super::pat::GateOr; use super::path::PathStyle; -use super::{BlockMode, Parser, Restrictions, SemiColonMode}; +use super::{BlockMode, Parser, Restrictions, SemiColonMode, SupportsCustomAttr}; use crate::maybe_whole; use rustc_ast as ast; @@ -33,49 +33,50 @@ impl<'a> Parser<'a> { fn parse_stmt_without_recovery(&mut self) -> PResult<'a, Option> { maybe_whole!(self, NtStmt, |x| Some(x)); - let (mut stmt, tokens) = self.parse_outer_attributes_with_tokens(|this, attrs| { - let lo = this.token.span; - - let stmt = if this.eat_keyword(kw::Let) { - this.parse_local_mk(lo, attrs.into())? - } else if this.is_kw_followed_by_ident(kw::Mut) { - this.recover_stmt_local(lo, attrs.into(), "missing keyword", "let mut")? - } else if this.is_kw_followed_by_ident(kw::Auto) { - this.bump(); // `auto` - let msg = "write `let` instead of `auto` to introduce a new variable"; - this.recover_stmt_local(lo, attrs.into(), msg, "let")? - } else if this.is_kw_followed_by_ident(sym::var) { - this.bump(); // `var` - let msg = "write `let` instead of `var` to introduce a new variable"; - this.recover_stmt_local(lo, attrs.into(), msg, "let")? - } else if this.check_path() - && !this.token.is_qpath_start() - && !this.is_path_start_item() - { - // We have avoided contextual keywords like `union`, items with `crate` visibility, - // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something - // that starts like a path (1 token), but it fact not a path. - // Also, we avoid stealing syntax from `parse_item_`. - this.parse_stmt_path_start(lo, attrs)? - } else if let Some(item) = - this.parse_item_common(attrs.clone(), false, true, |_| true)? - { - // FIXME: Bad copy of attrs - this.mk_stmt(lo.to(item.span), StmtKind::Item(P(item))) - } else if this.eat(&token::Semi) { - // Do not attempt to parse an expression if we're done here. - this.error_outer_attrs(&attrs); - this.mk_stmt(lo, StmtKind::Empty) - } else if this.token != token::CloseDelim(token::Brace) { - // Remainder are line-expr stmts. - let e = this.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs.into()))?; - this.mk_stmt(lo.to(e.span), StmtKind::Expr(e)) - } else { - this.error_outer_attrs(&attrs); - return Ok(None); - }; - Ok(Some(stmt)) - })?; + let (mut stmt, tokens) = + self.parse_outer_attributes_with_tokens(SupportsCustomAttr::Yes, |this, attrs| { + let lo = this.token.span; + + let stmt = if this.eat_keyword(kw::Let) { + this.parse_local_mk(lo, attrs.into())? + } else if this.is_kw_followed_by_ident(kw::Mut) { + this.recover_stmt_local(lo, attrs.into(), "missing keyword", "let mut")? + } else if this.is_kw_followed_by_ident(kw::Auto) { + this.bump(); // `auto` + let msg = "write `let` instead of `auto` to introduce a new variable"; + this.recover_stmt_local(lo, attrs.into(), msg, "let")? + } else if this.is_kw_followed_by_ident(sym::var) { + this.bump(); // `var` + let msg = "write `let` instead of `var` to introduce a new variable"; + this.recover_stmt_local(lo, attrs.into(), msg, "let")? + } else if this.check_path() + && !this.token.is_qpath_start() + && !this.is_path_start_item() + { + // We have avoided contextual keywords like `union`, items with `crate` visibility, + // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something + // that starts like a path (1 token), but it fact not a path. + // Also, we avoid stealing syntax from `parse_item_`. + this.parse_stmt_path_start(lo, attrs)? + } else if let Some(item) = + this.parse_item_common(attrs.clone(), false, true, |_| true)? + { + // FIXME: Bad copy of attrs + this.mk_stmt(lo.to(item.span), StmtKind::Item(P(item))) + } else if this.eat(&token::Semi) { + // Do not attempt to parse an expression if we're done here. + this.error_outer_attrs(&attrs); + this.mk_stmt(lo, StmtKind::Empty) + } else if this.token != token::CloseDelim(token::Brace) { + // Remainder are line-expr stmts. + let e = this.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs.into()))?; + this.mk_stmt(lo.to(e.span), StmtKind::Expr(e)) + } else { + this.error_outer_attrs(&attrs); + return Ok(None); + }; + Ok(Some(stmt)) + })?; match stmt.as_mut() { Some(Stmt { kind: StmtKind::Local(local), .. }) => local.tokens = tokens, Some(Stmt { kind: StmtKind::Item(item), .. }) => item.tokens = tokens,