Only collect tokens for attributes themselves when needed

We need attribute tokens if the target supports custom attributes, or we
are inside a `derive` target. Otherwise, we don't need tokens for
individual tokens - any needed token collection will be handled further
up the stack.

compiler/rustc_parse/src/parser/attr.rs

@@ -22,6 +22,12 @@ pub(super) enum InnerAttrPolicy<'a> {
 const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &str = "an inner attribute is not \
                                                      permitted in this context";
 
+#[derive(Copy, Clone, Eq, PartialEq)]
+pub enum SupportsCustomAttr {
+    Yes,
+    No,
+}
+
 pub struct CfgAttrItem {
     pub item: ast::AttrItem,
     pub span: Span,
@@ -39,12 +45,15 @@ impl<'a> Parser<'a> {
         self.check(&token::Pound) || matches!(self.token.kind, token::DocComment(..))
     }
 
-    fn parse_outer_attributes_(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
+    fn parse_outer_attributes_(
+        &mut self,
+        custom: SupportsCustomAttr,
+    ) -> PResult<'a, Vec<ast::Attribute>> {
         let mut attrs: Vec<ast::Attribute> = Vec::new();
         let mut just_parsed_doc_comment = false;
 
         loop {
-            let (attr, tokens) = self.collect_tokens_keep_in_stream(false, |this| {
+            let mut parse_attr = |this: &mut Self| {
                 debug!("parse_outer_attributes: self.token={:?}", this.token);
                 if this.check(&token::Pound) {
                     let inner_error_reason = if just_parsed_doc_comment {
@@ -85,9 +94,21 @@ impl<'a> Parser<'a> {
                 } else {
                     Ok((None, Vec::new()))
                 }
-            })?;
+            };
+
+            // `in_derive` does not take into account the attributes we are currently parsing
+            // (which may contain a `derive`). This is fine - if a `derive` attribute
+            // can legally occur here, `custom` will be `SupportsCustomAttr::Yes`
+            let (attr, tokens) = if custom == SupportsCustomAttr::Yes || self.in_derive {
+                let (attr, tokens) = self.collect_tokens_keep_in_stream(false, parse_attr)?;
+                (attr, Some(tokens))
+            } else {
+                let (attr, _nested_attrs) = parse_attr(self)?;
+                (attr, None)
+            };
+
             if let Some(mut attr) = attr {
-                attr.tokens = Some(tokens.to_tokenstream());
+                attr.tokens = tokens.map(|t| t.to_tokenstream());
                 attrs.push(attr);
             } else {
                 break;
@@ -102,6 +123,7 @@ impl<'a> Parser<'a> {
     >(
         &mut self,
         already_parsed_attrs: Option<AttrVec>,
+        custom: SupportsCustomAttr,
         f: F,
     ) -> PResult<'a, (R, Option<PreexpTokenStream>)> {
         let in_derive = self.in_derive;
@@ -119,10 +141,17 @@ impl<'a> Parser<'a> {
 
                 let mut res = res?;
 
-                res.visit_attrs(|attrs| {
-                    new_attrs = attrs.clone();
-                });
-                Ok((res, new_attrs))
+                // `this.in_derive` does not take into account our new attributes
+                // (which may contain a `derive`). This is fine - if a `derive` attribute
+                // can legally occur here, `custom` will be `SupportsCustomAttr::Yes`
+                if custom == SupportsCustomAttr::Yes || this.in_derive {
+                    res.visit_attrs(|attrs| {
+                        new_attrs = attrs.clone();
+                    });
+                    Ok((res, new_attrs))
+                } else {
+                    Ok((res, Vec::new()))
+                }
             })?;
             Ok((res, Some(tokens)))
         };
@@ -142,7 +171,7 @@ impl<'a> Parser<'a> {
                 return Ok((f(self, AttrVec::new())?, None));
             }
 
-            let attrs = self.parse_outer_attributes_()?;
+            let attrs = self.parse_outer_attributes_(custom)?;
             if !needs_tokens(&attrs) {
                 return Ok((f(self, attrs.into())?, None));
             }
@@ -153,17 +182,19 @@ impl<'a> Parser<'a> {
 
     pub(super) fn parse_outer_attributes<R: HasAttrs>(
         &mut self,
+        custom: SupportsCustomAttr,
         f: impl FnOnce(&mut Self, Vec<ast::Attribute>) -> PResult<'a, R>,
     ) -> PResult<'a, R> {
-        self.parse_outer_attributes_with_tokens(f).map(|(res, _tokens)| res)
+        self.parse_outer_attributes_with_tokens(custom, f).map(|(res, _tokens)| res)
     }
 
     /// Parses attributes that appear before an item.
     pub(super) fn parse_outer_attributes_with_tokens<R: HasAttrs>(
         &mut self,
+        custom: SupportsCustomAttr,
         f: impl FnOnce(&mut Self, Vec<ast::Attribute>) -> PResult<'a, R>,
     ) -> PResult<'a, (R, Option<PreexpTokenStream>)> {
-        self.parse_or_use_outer_attributes(None, |this, attrs| f(this, attrs.into()))
+        self.parse_or_use_outer_attributes(None, custom, |this, attrs| f(this, attrs.into()))
     }
 
     /// Matches `attribute = # ! [ meta_item ]`.

compiler/rustc_parse/src/parser/expr.rs

@@ -1,7 +1,7 @@
 use super::pat::{GateOr, PARAM_EXPECTED};
 use super::ty::{AllowPlus, RecoverQPath};
 use super::{BlockMode, Parser, PathStyle, Restrictions, TokenType};
-use super::{SemiColonMode, SeqSep, TokenExpectType};
+use super::{SemiColonMode, SeqSep, SupportsCustomAttr, TokenExpectType};
 use crate::maybe_recover_from_interpolated_ty_qpath;
 
 use rustc_ast::ptr::P;
@@ -445,41 +445,43 @@ impl<'a> Parser<'a> {
             _ => RangeLimits::Closed,
         };
         let op = AssocOp::from_token(&self.token);
-        let (mut expr, tokens) = self.parse_or_use_outer_attributes(attrs, |this, attrs| {
-            let lo = this.token.span;
-            this.bump();
-            let (span, opt_end) = if this.is_at_start_of_range_notation_rhs() {
-                // RHS must be parsed with more associativity than the dots.
-                this.parse_assoc_expr_with(op.unwrap().precedence() + 1, LhsExpr::NotYetParsed)
-                    .map(|x| (lo.to(x.span), Some(x)))?
-            } else {
-                (lo, None)
-            };
-            Ok(this.mk_expr(span, this.mk_range(None, opt_end, limits)?, attrs))
-        })?;
+        let (mut expr, tokens) =
+            self.parse_or_use_outer_attributes(attrs, SupportsCustomAttr::Yes, |this, attrs| {
+                let lo = this.token.span;
+                this.bump();
+                let (span, opt_end) = if this.is_at_start_of_range_notation_rhs() {
+                    // RHS must be parsed with more associativity than the dots.
+                    this.parse_assoc_expr_with(op.unwrap().precedence() + 1, LhsExpr::NotYetParsed)
+                        .map(|x| (lo.to(x.span), Some(x)))?
+                } else {
+                    (lo, None)
+                };
+                Ok(this.mk_expr(span, this.mk_range(None, opt_end, limits)?, attrs))
+            })?;
         expr.tokens = tokens;
         Ok(expr)
     }
 
     /// Parses a prefix-unary-operator expr.
     fn parse_prefix_expr(&mut self, attrs: Option<AttrVec>) -> PResult<'a, P<Expr>> {
-        let (mut expr, tokens) = self.parse_or_use_outer_attributes(attrs, |this, attrs| {
-            let lo = this.token.span;
-            // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
-            let (hi, ex) = match this.token.uninterpolate().kind {
-                token::Not => this.parse_unary_expr(lo, UnOp::Not), // `!expr`
-                token::Tilde => this.recover_tilde_expr(lo),        // `~expr`
-                token::BinOp(token::Minus) => this.parse_unary_expr(lo, UnOp::Neg), // `-expr`
-                token::BinOp(token::Star) => this.parse_unary_expr(lo, UnOp::Deref), // `*expr`
-                token::BinOp(token::And) | token::AndAnd => this.parse_borrow_expr(lo),
-                token::Ident(..) if this.token.is_keyword(kw::Box) => this.parse_box_expr(lo),
-                token::Ident(..) if this.is_mistaken_not_ident_negation() => {
-                    this.recover_not_expr(lo)
-                }
-                _ => return this.parse_dot_or_call_expr(attrs),
-            }?;
-            Ok(this.mk_expr(lo.to(hi), ex, attrs))
-        })?;
+        let (mut expr, tokens) =
+            self.parse_or_use_outer_attributes(attrs, SupportsCustomAttr::Yes, |this, attrs| {
+                let lo = this.token.span;
+                // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
+                let (hi, ex) = match this.token.uninterpolate().kind {
+                    token::Not => this.parse_unary_expr(lo, UnOp::Not), // `!expr`
+                    token::Tilde => this.recover_tilde_expr(lo),        // `~expr`
+                    token::BinOp(token::Minus) => this.parse_unary_expr(lo, UnOp::Neg), // `-expr`
+                    token::BinOp(token::Star) => this.parse_unary_expr(lo, UnOp::Deref), // `*expr`
+                    token::BinOp(token::And) | token::AndAnd => this.parse_borrow_expr(lo),
+                    token::Ident(..) if this.token.is_keyword(kw::Box) => this.parse_box_expr(lo),
+                    token::Ident(..) if this.is_mistaken_not_ident_negation() => {
+                        this.recover_not_expr(lo)
+                    }
+                    _ => return this.parse_dot_or_call_expr(attrs),
+                }?;
+                Ok(this.mk_expr(lo.to(hi), ex, attrs))
+            })?;
         expr.tokens = tokens;
         Ok(expr)
     }
@@ -1598,7 +1600,7 @@ impl<'a> Parser<'a> {
     /// Parses a parameter in a closure header (e.g., `|arg, arg|`).
     fn parse_fn_block_param(&mut self) -> PResult<'a, Param> {
         let lo = self.token.span;
-        self.parse_outer_attributes(|this, attrs| {
+        self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| {
             let pat = this.parse_pat(PARAM_EXPECTED)?;
             let ty = if this.eat(&token::Colon) {
                 this.parse_ty()?
@@ -1629,7 +1631,8 @@ impl<'a> Parser<'a> {
             self.error_missing_if_cond(lo, cond.span)
         } else {
             // For recovery.
-            let attrs = self.parse_outer_attributes(|_this, attrs| Ok(attrs))?;
+            let attrs =
+                self.parse_outer_attributes(SupportsCustomAttr::No, |_this, attrs| Ok(attrs))?;
             let not_block = self.token != token::OpenDelim(token::Brace);
             let block = self.parse_block().map_err(|mut err| {
                 if not_block {
@@ -1686,7 +1689,7 @@ impl<'a> Parser<'a> {
     /// Parses an `else { ... }` expression (`else` token already eaten).
     fn parse_else_expr(&mut self) -> PResult<'a, P<Expr>> {
         let ctx_span = self.prev_token.span; // `else`
-        self.parse_outer_attributes(|this, attrs| {
+        self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| {
             let expr = if this.eat_keyword(kw::If) {
                 this.parse_if_expr(AttrVec::new())?
             } else {
@@ -1845,7 +1848,7 @@ impl<'a> Parser<'a> {
     }
 
     pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> {
-        self.parse_outer_attributes(|this, attrs| {
+        self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| {
             let lo = this.token.span;
             let pat = this.parse_top_pat(GateOr::No)?;
             let guard = if this.eat_keyword(kw::If) {
@@ -2161,7 +2164,7 @@ impl<'a> Parser<'a> {
 
     /// Parses `ident (COLON expr)?`.
     fn parse_field(&mut self) -> PResult<'a, Field> {
-        self.parse_outer_attributes(|this, attrs| {
+        self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| {
             let attrs = attrs.into();
             let lo = this.token.span;
 

compiler/rustc_parse/src/parser/generics.rs

@@ -1,4 +1,4 @@
-use super::Parser;
+use super::{Parser, SupportsCustomAttr};
 
 use rustc_ast::token;
 use rustc_ast::{
@@ -74,7 +74,7 @@ impl<'a> Parser<'a> {
         let mut params = Vec::new();
         loop {
             let mut should_break = false;
-            let param = self.parse_outer_attributes(|this, attrs| {
+            let param = self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| {
                 let param = if this.check_lifetime() {
                     let lifetime = this.expect_lifetime();
                     // Parse lifetime parameter.

compiler/rustc_parse/src/parser/item.rs

@@ -1,6 +1,6 @@
 use super::diagnostics::{dummy_arg, ConsumeClosingDelim, Error};
 use super::ty::{AllowPlus, RecoverQPath};
-use super::{FollowedByType, Parser, PathStyle};
+use super::{FollowedByType, Parser, PathStyle, SupportsCustomAttr};
 
 use crate::maybe_whole;
 use crate::parser::attr::attrs_require_tokens;
@@ -101,9 +101,10 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_item_(&mut self, req_name: ReqName) -> PResult<'a, Option<Item>> {
-        let res = self.parse_outer_attributes_with_tokens(|this, attrs| {
-            this.parse_item_common(attrs, true, false, req_name)
-        });
+        let res = self
+            .parse_outer_attributes_with_tokens(SupportsCustomAttr::Yes, |this, attrs| {
+                this.parse_item_common(attrs, true, false, req_name)
+            });
         res.map(|(mut item, tokens)| {
             if let Some(item) = item.as_mut() {
                 if item.tokens.is_none() {
@@ -1073,7 +1074,7 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_enum_variant(&mut self) -> PResult<'a, Option<Variant>> {
-        self.parse_outer_attributes(|this, variant_attrs| {
+        self.parse_outer_attributes(SupportsCustomAttr::No, |this, variant_attrs| {
             let vlo = this.token.span;
 
             let vis = this.parse_visibility(FollowedByType::No)?;
@@ -1259,7 +1260,7 @@ impl<'a> Parser<'a> {
         // This is the case where we find `struct Foo<T>(T) where T: Copy;`
         // Unit like structs are handled in parse_item_struct function
         self.parse_paren_comma_seq(|p| {
-            p.parse_outer_attributes(|p, attrs| {
+            p.parse_outer_attributes(SupportsCustomAttr::No, |p, attrs| {
                 let lo = p.token.span;
                 let vis = p.parse_visibility(FollowedByType::Yes)?;
                 let ty = p.parse_ty()?;
@@ -1279,7 +1280,7 @@ impl<'a> Parser<'a> {
 
     /// Parses an element of a struct declaration.
     fn parse_struct_decl_field(&mut self) -> PResult<'a, StructField> {
-        self.parse_outer_attributes(|this, attrs| {
+        self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| {
             let lo = this.token.span;
             let vis = this.parse_visibility(FollowedByType::No)?;
             this.parse_single_struct_field(lo, vis, attrs)
@@ -1720,7 +1721,7 @@ impl<'a> Parser<'a> {
     /// - `self` is syntactically allowed when `first_param` holds.
     fn parse_param_general(&mut self, req_name: ReqName, first_param: bool) -> PResult<'a, Param> {
         let lo = self.token.span;
-        self.parse_outer_attributes(|this, attrs| {
+        self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| {
             // Possibly parse `self`. Recover if we parsed it and it wasn't allowed here.
             if let Some(mut param) = this.parse_self_param()? {
                 param.attrs = attrs.into();
@@ -1908,7 +1909,7 @@ impl<'a> Parser<'a> {
 
     fn recover_first_param(&mut self) -> &'static str {
         let res = self
-            .parse_outer_attributes(|this, _attrs| this.parse_self_param())
+            .parse_outer_attributes(SupportsCustomAttr::No, |this, _attrs| this.parse_self_param())
             .map_err(|mut err| err.cancel());
         match res {
             Ok(Some(_)) => "method",

compiler/rustc_parse/src/parser/mod.rs

@@ -10,6 +10,7 @@ mod stmt;
 mod ty;
 
 use crate::lexer::UnmatchedBrace;
+pub use attr::SupportsCustomAttr;
 use diagnostics::Error;
 pub use path::PathStyle;
 
@@ -27,9 +28,8 @@ use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError
 use rustc_session::parse::ParseSess;
 use rustc_span::source_map::{Span, DUMMY_SP};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use tracing::debug;
-
 use std::{cmp, mem, slice};
+use tracing::debug;
 
 bitflags::bitflags! {
     struct Restrictions: u8 {

compiler/rustc_parse/src/parser/pat.rs

@@ -1,4 +1,4 @@
-use super::{Parser, PathStyle};
+use super::{Parser, PathStyle, SupportsCustomAttr};
 use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole};
 use rustc_ast::mut_visit::{noop_visit_mac, noop_visit_pat, MutVisitor};
 use rustc_ast::ptr::P;
@@ -831,7 +831,7 @@ impl<'a> Parser<'a> {
         let mut etc_span = None;
 
         while self.token != token::CloseDelim(token::Brace) {
-            let field_pat = self.parse_outer_attributes(|this, attrs| {
+            let field_pat = self.parse_outer_attributes(SupportsCustomAttr::No, |this, attrs| {
                 let lo = this.token.span;
 
                 // check that a comma comes after every field