Auto merge of rust-lang#123865 - eholk:expr_2021, r=fmease

Update `expr` matcher for Edition 2024 and add `expr_2021` nonterminal

This commit adds a new nonterminal `expr_2021` in macro patterns, and `expr_fragment_specifier_2024` feature flag.

This change also updates `expr` so that on Edition 2024 it will also match `const { ... }` blocks, while `expr_2021` preserves the current behavior of `expr`, matching expressions without `const` blocks.

Joint work with `@vincenzopalazzo.`

Issue rust-lang#123742

compiler/rustc_ast/src/token.rs

@@ -881,6 +881,8 @@ pub enum NonterminalKind {
     },
     PatWithOr,
     Expr,
+    /// Matches an expression using the rules from edition 2021 and earlier.
+    Expr2021,
     Ty,
     Ident,
     Lifetime,
@@ -910,6 +912,7 @@ impl NonterminalKind {
             },
             sym::pat_param => NonterminalKind::PatParam { inferred: false },
             sym::expr => NonterminalKind::Expr,
+            sym::expr_2021 if edition().at_least_rust_2021() => NonterminalKind::Expr2021,
             sym::ty => NonterminalKind::Ty,
             sym::ident => NonterminalKind::Ident,
             sym::lifetime => NonterminalKind::Lifetime,
@@ -929,6 +932,7 @@ impl NonterminalKind {
             NonterminalKind::PatParam { inferred: false } => sym::pat_param,
             NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat,
             NonterminalKind::Expr => sym::expr,
+            NonterminalKind::Expr2021 => sym::expr_2021,
             NonterminalKind::Ty => sym::ty,
             NonterminalKind::Ident => sym::ident,
             NonterminalKind::Lifetime => sym::lifetime,

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -1290,7 +1290,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
                 // maintain
                 IsInFollow::Yes
             }
-            NonterminalKind::Stmt | NonterminalKind::Expr => {
+            NonterminalKind::Stmt | NonterminalKind::Expr | NonterminalKind::Expr2021 => {
                 const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"];
                 match tok {
                     TokenTree::Token(token) => match token.kind {

compiler/rustc_expand/src/mbe/quoted.rs

@@ -16,6 +16,10 @@ use rustc_span::Span;
 const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \
                                         `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \
                                         `literal`, `path`, `meta`, `tt`, `item` and `vis`";
+const VALID_FRAGMENT_NAMES_MSG_2021: &str = "valid fragment specifiers are \
+                                             `ident`, `block`, `stmt`, `expr`, `expr_2021`, `pat`, \
+                                             `ty`, `lifetime`, `literal`, `path`, `meta`, `tt`, \
+                                             `item` and `vis`";
 
 /// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this
 /// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a
@@ -63,35 +67,60 @@ pub(super) fn parse(
                             Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() {
                                 Some((fragment, _)) => {
                                     let span = token.span.with_lo(start_sp.lo());
-
+                                    let edition = || {
+                                        // FIXME(#85708) - once we properly decode a foreign
+                                        // crate's `SyntaxContext::root`, then we can replace
+                                        // this with just `span.edition()`. A
+                                        // `SyntaxContext::root()` from the current crate will
+                                        // have the edition of the current crate, and a
+                                        // `SyntaxContext::root()` from a foreign crate will
+                                        // have the edition of that crate (which we manually
+                                        // retrieve via the `edition` parameter).
+                                        if !span.from_expansion() {
+                                            edition
+                                        } else {
+                                            span.edition()
+                                        }
+                                    };
                                     let kind =
-                                        token::NonterminalKind::from_symbol(fragment.name, || {
-                                            // FIXME(#85708) - once we properly decode a foreign
-                                            // crate's `SyntaxContext::root`, then we can replace
-                                            // this with just `span.edition()`. A
-                                            // `SyntaxContext::root()` from the current crate will
-                                            // have the edition of the current crate, and a
-                                            // `SyntaxContext::root()` from a foreign crate will
-                                            // have the edition of that crate (which we manually
-                                            // retrieve via the `edition` parameter).
-                                            if !span.from_expansion() {
-                                                edition
-                                            } else {
-                                                span.edition()
-                                            }
-                                        })
-                                        .unwrap_or_else(
-                                            || {
+                                        token::NonterminalKind::from_symbol(fragment.name, edition)
+                                            .unwrap_or_else(|| {
+                                                let help = match fragment.name {
+                                                    sym::expr_2021 => {
+                                                        format!(
+                                                            "fragment specifier `expr_2021` \
+                                                             requires Rust 2021 or later\n\
+                                                             {VALID_FRAGMENT_NAMES_MSG}"
+                                                        )
+                                                    }
+                                                    _ if edition().at_least_rust_2021()
+                                                        && features
+                                                            .expr_fragment_specifier_2024 =>
+                                                    {
+                                                        VALID_FRAGMENT_NAMES_MSG_2021.into()
+                                                    }
+                                                    _ => VALID_FRAGMENT_NAMES_MSG.into(),
+                                                };
                                                 sess.dcx().emit_err(
                                                     errors::InvalidFragmentSpecifier {
                                                         span,
                                                         fragment,
-                                                        help: VALID_FRAGMENT_NAMES_MSG.into(),
+                                                        help,
                                                     },
                                                 );
                                                 token::NonterminalKind::Ident
-                                            },
-                                        );
+                                            });
+                                    if kind == token::NonterminalKind::Expr2021
+                                        && !features.expr_fragment_specifier_2024
+                                    {
+                                        rustc_session::parse::feature_err(
+                                            sess,
+                                            sym::expr_fragment_specifier_2024,
+                                            span,
+                                            "fragment specifier `expr_2021` is unstable",
+                                        )
+                                        .emit();
+                                    }
                                     result.push(TokenTree::MetaVarDecl(span, ident, Some(kind)));
                                     continue;
                                 }

compiler/rustc_feature/src/unstable.rs

@@ -458,6 +458,8 @@ declare_features! (
     (unstable, exhaustive_patterns, "1.13.0", Some(51085)),
     /// Allows explicit tail calls via `become` expression.
     (incomplete, explicit_tail_calls, "1.72.0", Some(112788)),
+    /// Uses 2024 rules for matching `expr` fragments in macros. Also enables `expr_2021` fragment.
+    (incomplete, expr_fragment_specifier_2024, "CURRENT_RUSTC_VERSION", Some(123742)),
     /// Allows using `efiapi`, `sysv64` and `win64` as calling convention
     /// for functions with varargs.
     (unstable, extended_varargs_abi_support, "1.65.0", Some(100189)),

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -36,13 +36,19 @@ impl<'a> Parser<'a> {
         }
 
         match kind {
-            NonterminalKind::Expr => {
+            NonterminalKind::Expr2021 => {
                 token.can_begin_expr()
                 // This exception is here for backwards compatibility.
                 && !token.is_keyword(kw::Let)
                 // This exception is here for backwards compatibility.
                 && !token.is_keyword(kw::Const)
             }
+            NonterminalKind::Expr => {
+                token.can_begin_expr()
+                // This exception is here for backwards compatibility.
+                && !token.is_keyword(kw::Let)
+                && (token.span.edition().at_least_rust_2024() || !token.is_keyword(kw::Const))
+            }
             NonterminalKind::Ty => token.can_begin_type(),
             NonterminalKind::Ident => get_macro_ident(token).is_some(),
             NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
@@ -143,7 +149,9 @@ impl<'a> Parser<'a> {
                 })?)
             }
 
-            NonterminalKind::Expr => NtExpr(self.parse_expr_force_collect()?),
+            NonterminalKind::Expr | NonterminalKind::Expr2021 => {
+                NtExpr(self.parse_expr_force_collect()?)
+            }
             NonterminalKind::Literal => {
                 // The `:literal` matcher does not support attributes
                 NtLiteral(self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?)

compiler/rustc_span/src/symbol.rs

@@ -782,6 +782,8 @@ symbols! {
         explicit_tail_calls,
         export_name,
         expr,
+        expr_2021,
+        expr_fragment_specifier_2024,
         extended_key_value_attributes,
         extended_varargs_abi_support,
         extern_absolute_paths,

tests/ui/macros/expr_2021_inline_const.edi2021.stderr

@@ -0,0 +1,32 @@
+error: no rules expected the token `const`
+  --> $DIR/expr_2021_inline_const.rs:21:12
+   |
+LL | macro_rules! m2021 {
+   | ------------------ when calling this macro
+...
+LL |     m2021!(const { 1 });
+   |            ^^^^^ no rules expected this token in macro call
+   |
+note: while trying to match meta-variable `$e:expr_2021`
+  --> $DIR/expr_2021_inline_const.rs:10:6
+   |
+LL |     ($e:expr_2021) => {
+   |      ^^^^^^^^^^^^
+
+error: no rules expected the token `const`
+  --> $DIR/expr_2021_inline_const.rs:22:12
+   |
+LL | macro_rules! m2024 {
+   | ------------------ when calling this macro
+...
+LL |     m2024!(const { 1 });
+   |            ^^^^^ no rules expected this token in macro call
+   |
+note: while trying to match meta-variable `$e:expr`
+  --> $DIR/expr_2021_inline_const.rs:16:6
+   |
+LL |     ($e:expr) => {
+   |      ^^^^^^^
+
+error: aborting due to 2 previous errors
+

tests/ui/macros/expr_2021_inline_const.edi2024.stderr

@@ -0,0 +1,17 @@
+error: no rules expected the token `const`
+  --> $DIR/expr_2021_inline_const.rs:21:12
+   |
+LL | macro_rules! m2021 {
+   | ------------------ when calling this macro
+...
+LL |     m2021!(const { 1 });
+   |            ^^^^^ no rules expected this token in macro call
+   |
+note: while trying to match meta-variable `$e:expr_2021`
+  --> $DIR/expr_2021_inline_const.rs:10:6
+   |
+LL |     ($e:expr_2021) => {
+   |      ^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+

tests/ui/macros/expr_2021_inline_const.rs

@@ -0,0 +1,23 @@
+//@ revisions: edi2021 edi2024
+//@[edi2024]compile-flags: --edition=2024 -Z unstable-options
+//@[edi2021]compile-flags: --edition=2021
+
+// This test ensures that the inline const match only on edition 2024
+#![feature(expr_fragment_specifier_2024)]
+#![allow(incomplete_features)]
+
+macro_rules! m2021 {
+    ($e:expr_2021) => {
+        $e
+    };
+}
+
+macro_rules! m2024 {
+    ($e:expr) => {
+        $e
+    };
+}
+fn main() {
+    m2021!(const { 1 }); //~ ERROR: no rules expected the token `const`
+    m2024!(const { 1 }); //[edi2021]~ ERROR: no rules expected the token `const`
+}

tests/ui/macros/expr_2021_old_edition.rs

@@ -0,0 +1,13 @@
+//@  compile-flags: --edition=2018
+
+// This test ensures that expr_2021 is not allowed on pre-2021 editions
+
+macro_rules! m {
+    ($e:expr_2021) => { //~ ERROR: invalid fragment specifier `expr_2021`
+        $e
+    };
+}
+
+fn main() {
+    m!(()); //~ ERROR: no rules expected the token `(`
+}

tests/ui/macros/expr_2021_old_edition.stderr

@@ -0,0 +1,26 @@
+error: invalid fragment specifier `expr_2021`
+  --> $DIR/expr_2021_old_edition.rs:6:6
+   |
+LL |     ($e:expr_2021) => {
+   |      ^^^^^^^^^^^^
+   |
+   = help: fragment specifier `expr_2021` requires Rust 2021 or later
+           valid fragment specifiers are `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, `literal`, `path`, `meta`, `tt`, `item` and `vis`
+
+error: no rules expected the token `(`
+  --> $DIR/expr_2021_old_edition.rs:12:8
+   |
+LL | macro_rules! m {
+   | -------------- when calling this macro
+...
+LL |     m!(());
+   |        ^ no rules expected this token in macro call
+   |
+note: while trying to match meta-variable `$e:ident`
+  --> $DIR/expr_2021_old_edition.rs:6:6
+   |
+LL |     ($e:expr_2021) => {
+   |      ^^^^^^^^^^^^
+
+error: aborting due to 2 previous errors
+

tests/ui/macros/feature-gate-expr_fragment_specifier_2024.rs

@@ -0,0 +1,11 @@
+//@  compile-flags: --edition=2024 -Z unstable-options
+
+macro_rules! m {
+    ($e:expr_2021) => { //~ ERROR: fragment specifier `expr_2021` is unstable
+        $e
+    };
+}
+
+fn main() {
+    m!(());
+}

tests/ui/macros/feature-gate-expr_fragment_specifier_2024.stderr

@@ -0,0 +1,13 @@
+error[E0658]: fragment specifier `expr_2021` is unstable
+  --> $DIR/feature-gate-expr_fragment_specifier_2024.rs:4:6
+   |
+LL |     ($e:expr_2021) => {
+   |      ^^^^^^^^^^^^
+   |
+   = note: see issue #123742 <https://github.com/rust-lang/rust/issues/123742> for more information
+   = help: add `#![feature(expr_fragment_specifier_2024)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0658`.