diff --git a/src/data.rs b/src/data.rs index 6c76887516..96db2a0b7c 100644 --- a/src/data.rs +++ b/src/data.rs @@ -248,6 +248,8 @@ pub(crate) mod parsing { use crate::parse::discouraged::Speculative as _; use crate::parse::{Parse, ParseStream}; use crate::restriction::{FieldMutability, Visibility}; + #[cfg(not(feature = "full"))] + use crate::scan_expr::scan_expr; use crate::token; use crate::ty::Type; use crate::verbatim; @@ -276,7 +278,7 @@ pub(crate) mod parsing { let mut discriminant: Result = ahead.parse(); if discriminant.is_ok() { input.advance_to(&ahead); - } else if scan_lenient_discriminant(input).is_ok() { + } else if scan_expr(input).is_ok() { discriminant = Ok(Expr::Verbatim(verbatim::between(&begin, input))); } discriminant? @@ -294,85 +296,6 @@ pub(crate) mod parsing { } } - #[cfg(not(feature = "full"))] - pub(crate) fn scan_lenient_discriminant(input: ParseStream) -> Result<()> { - use crate::expr::Member; - use crate::lifetime::Lifetime; - use crate::lit::Lit; - use crate::lit::LitFloat; - use crate::op::{BinOp, UnOp}; - use crate::path::{self, AngleBracketedGenericArguments}; - use proc_macro2::Delimiter::{self, Brace, Bracket, Parenthesis}; - - let consume = |delimiter: Delimiter| { - Result::unwrap(input.step(|cursor| match cursor.group(delimiter) { - Some((_inside, _span, rest)) => Ok((true, rest)), - None => Ok((false, *cursor)), - })) - }; - - macro_rules! consume { - [$token:tt] => { - input.parse::>().unwrap().is_some() - }; - } - - let mut initial = true; - let mut depth = 0usize; - loop { - if initial { - if consume![&] { - initial = consume![mut] || !consume![raw] || consume![const] || consume![mut]; - } else if consume![if] || consume![match] || consume![while] { - depth += 1; - } else if input.parse::>()?.is_some() - || (consume(Brace) || consume(Bracket) || consume(Parenthesis)) - || (consume![async] || consume![const] || consume![loop] || consume![unsafe]) - && (consume(Brace) || break) - { - initial = false; - } else if consume![let] { - while !consume![=] { - if !((consume![|] || consume![ref] || consume![mut] || consume![@]) - || (consume![!] || input.parse::>()?.is_some()) - || (consume![..=] || consume![..] || consume![&] || consume![_]) - || (consume(Brace) || consume(Bracket) || consume(Parenthesis))) - { - path::parsing::qpath(input, true)?; - } - } - } else if input.parse::>()?.is_some() && !consume![:] { - break; - } else if input.parse::().is_err() { - path::parsing::qpath(input, true)?; - initial = consume![!] || depth == 0 && input.peek(token::Brace); - } - } else if input.is_empty() || input.peek(Token![,]) { - return Ok(()); - } else if depth > 0 && consume(Brace) { - if consume![else] && !consume(Brace) { - initial = consume![if] || break; - } else { - depth -= 1; - } - } else if input.parse::().is_ok() || (consume![..] | consume![=]) { - initial = true; - } else if consume![.] { - if input.parse::>()?.is_none() - && (input.parse::()?.is_named() && consume![::]) - { - AngleBracketedGenericArguments::do_parse(None, input)?; - } - } else if consume![as] { - input.parse::()?; - } else if !(consume(Brace) || consume(Bracket) || consume(Parenthesis)) { - break; - } - } - - Err(input.error("unsupported expression")) - } - #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] impl Parse for FieldsNamed { fn parse(input: ParseStream) -> Result { diff --git a/src/lib.rs b/src/lib.rs index 2eb370ad92..d97f3b0477 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -264,6 +264,7 @@ clippy::derivable_impls, clippy::diverging_sub_expression, clippy::doc_markdown, + clippy::enum_glob_use, clippy::expl_impl_clone_on_copy, clippy::explicit_auto_deref, clippy::if_not_else, @@ -307,6 +308,8 @@ clippy::wildcard_imports, )] +extern crate self as syn; + #[cfg(feature = "proc-macro")] extern crate proc_macro; @@ -509,6 +512,9 @@ pub use crate::restriction::{FieldMutability, VisRestricted, Visibility}; mod sealed; +#[cfg(all(feature = "parsing", feature = "derive", not(feature = "full")))] +mod scan_expr; + mod span; #[cfg(all(feature = "parsing", feature = "printing"))] diff --git a/src/scan_expr.rs b/src/scan_expr.rs new file mode 100644 index 0000000000..155b5b63bf --- /dev/null +++ b/src/scan_expr.rs @@ -0,0 +1,264 @@ +use self::{Action::*, Input::*}; +use proc_macro2::{Delimiter, Ident, Spacing, TokenTree}; +use syn::parse::{ParseStream, Result}; +use syn::{AngleBracketedGenericArguments, BinOp, Expr, ExprPath, Lifetime, Lit, Token, Type}; + +enum Input { + Keyword(&'static str), + Punct(&'static str), + ConsumeAny, + ConsumeBinOp, + ConsumeBrace, + ConsumeDelimiter, + ConsumeIdent, + ConsumeLifetime, + ConsumeLiteral, + ConsumeNestedBrace, + ExpectPath, + ExpectTurbofish, + ExpectType, + CanBeginExpr, + Otherwise, + Empty, +} + +enum Action { + SetState(&'static [(Input, Action)]), + IncDepth, + DecDepth, + Finish, +} + +static INIT: [(Input, Action); 28] = [ + (ConsumeDelimiter, SetState(&POSTFIX)), + (Keyword("async"), SetState(&ASYNC)), + (Keyword("break"), SetState(&BREAK_LABEL)), + (Keyword("const"), SetState(&CONST)), + (Keyword("continue"), SetState(&CONTINUE)), + (Keyword("for"), SetState(&FOR)), + (Keyword("if"), IncDepth), + (Keyword("let"), SetState(&PATTERN)), + (Keyword("loop"), SetState(&BLOCK)), + (Keyword("match"), IncDepth), + (Keyword("move"), SetState(&CLOSURE)), + (Keyword("return"), SetState(&RETURN)), + (Keyword("static"), SetState(&CLOSURE)), + (Keyword("unsafe"), SetState(&BLOCK)), + (Keyword("while"), IncDepth), + (Keyword("yield"), SetState(&RETURN)), + (Keyword("_"), SetState(&POSTFIX)), + (Punct("!"), SetState(&INIT)), + (Punct("#"), SetState(&[(ConsumeDelimiter, SetState(&INIT))])), + (Punct("&"), SetState(&REFERENCE)), + (Punct("*"), SetState(&INIT)), + (Punct("-"), SetState(&INIT)), + (Punct("..="), SetState(&INIT)), + (Punct(".."), SetState(&RANGE)), + (Punct("|"), SetState(&CLOSURE_ARGS)), + (ConsumeLifetime, SetState(&[(Punct(":"), SetState(&INIT))])), + (ConsumeLiteral, SetState(&POSTFIX)), + (ExpectPath, SetState(&PATH)), +]; + +static POSTFIX: [(Input, Action); 10] = [ + (Keyword("as"), SetState(&[(ExpectType, SetState(&POSTFIX))])), + (Punct("..="), SetState(&INIT)), + (Punct(".."), SetState(&RANGE)), + (Punct("."), SetState(&DOT)), + (Punct("?"), SetState(&POSTFIX)), + (ConsumeBinOp, SetState(&INIT)), + (Punct("="), SetState(&INIT)), + (ConsumeNestedBrace, SetState(&IF_THEN)), + (ConsumeDelimiter, SetState(&POSTFIX)), + (Empty, Finish), +]; + +static ASYNC: [(Input, Action); 3] = [ + (Keyword("move"), SetState(&ASYNC)), + (Punct("|"), SetState(&CLOSURE_ARGS)), + (ConsumeBrace, SetState(&POSTFIX)), +]; + +static BLOCK: [(Input, Action); 1] = [(ConsumeBrace, SetState(&POSTFIX))]; + +static BREAK_LABEL: [(Input, Action); 2] = [ + (ConsumeLifetime, SetState(&BREAK_VALUE)), + (Otherwise, SetState(&BREAK_VALUE)), +]; + +static BREAK_VALUE: [(Input, Action); 3] = [ + (ConsumeNestedBrace, SetState(&IF_THEN)), + (CanBeginExpr, SetState(&INIT)), + (Otherwise, SetState(&POSTFIX)), +]; + +static CLOSURE: [(Input, Action); 6] = [ + (Keyword("async"), SetState(&CLOSURE)), + (Keyword("move"), SetState(&CLOSURE)), + (Punct(","), SetState(&CLOSURE)), + (Punct(">"), SetState(&CLOSURE)), + (Punct("|"), SetState(&CLOSURE_ARGS)), + (ConsumeLifetime, SetState(&CLOSURE)), +]; + +static CLOSURE_ARGS: [(Input, Action); 2] = [ + (Punct("|"), SetState(&CLOSURE_RET)), + (ConsumeAny, SetState(&CLOSURE_ARGS)), +]; + +static CLOSURE_RET: [(Input, Action); 2] = [ + (Punct("->"), SetState(&[(ExpectType, SetState(&BLOCK))])), + (Otherwise, SetState(&INIT)), +]; + +static CONST: [(Input, Action); 2] = [ + (Punct("|"), SetState(&CLOSURE_ARGS)), + (ConsumeBrace, SetState(&POSTFIX)), +]; + +static CONTINUE: [(Input, Action); 2] = [ + (ConsumeLifetime, SetState(&POSTFIX)), + (Otherwise, SetState(&POSTFIX)), +]; + +static DOT: [(Input, Action); 3] = [ + (Keyword("await"), SetState(&POSTFIX)), + (ConsumeIdent, SetState(&METHOD)), + (ConsumeLiteral, SetState(&POSTFIX)), +]; + +static FOR: [(Input, Action); 2] = [ + (Punct("<"), SetState(&CLOSURE)), + (Otherwise, SetState(&PATTERN)), +]; + +static IF_ELSE: [(Input, Action); 2] = [(Keyword("if"), SetState(&INIT)), (ConsumeBrace, DecDepth)]; +static IF_THEN: [(Input, Action); 2] = + [(Keyword("else"), SetState(&IF_ELSE)), (Otherwise, DecDepth)]; + +static METHOD: [(Input, Action); 1] = [(ExpectTurbofish, SetState(&POSTFIX))]; + +static PATH: [(Input, Action); 4] = [ + (Punct("!="), SetState(&INIT)), + (Punct("!"), SetState(&INIT)), + (ConsumeNestedBrace, SetState(&IF_THEN)), + (Otherwise, SetState(&POSTFIX)), +]; + +static PATTERN: [(Input, Action); 15] = [ + (ConsumeDelimiter, SetState(&PATTERN)), + (Keyword("box"), SetState(&PATTERN)), + (Keyword("in"), IncDepth), + (Keyword("mut"), SetState(&PATTERN)), + (Keyword("ref"), SetState(&PATTERN)), + (Keyword("_"), SetState(&PATTERN)), + (Punct("!"), SetState(&PATTERN)), + (Punct("&"), SetState(&PATTERN)), + (Punct("..="), SetState(&PATTERN)), + (Punct(".."), SetState(&PATTERN)), + (Punct("="), SetState(&INIT)), + (Punct("@"), SetState(&PATTERN)), + (Punct("|"), SetState(&PATTERN)), + (ConsumeLiteral, SetState(&PATTERN)), + (ExpectPath, SetState(&PATTERN)), +]; + +static RANGE: [(Input, Action); 6] = [ + (Punct("..="), SetState(&INIT)), + (Punct(".."), SetState(&RANGE)), + (Punct("."), SetState(&DOT)), + (ConsumeNestedBrace, SetState(&IF_THEN)), + (Empty, Finish), + (Otherwise, SetState(&INIT)), +]; + +static RAW: [(Input, Action); 3] = [ + (Keyword("const"), SetState(&INIT)), + (Keyword("mut"), SetState(&INIT)), + (Otherwise, SetState(&POSTFIX)), +]; + +static REFERENCE: [(Input, Action); 3] = [ + (Keyword("mut"), SetState(&INIT)), + (Keyword("raw"), SetState(&RAW)), + (Otherwise, SetState(&INIT)), +]; + +static RETURN: [(Input, Action); 2] = [ + (CanBeginExpr, SetState(&INIT)), + (Otherwise, SetState(&POSTFIX)), +]; + +pub(crate) fn scan_expr(input: ParseStream) -> Result<()> { + let mut state = INIT.as_slice(); + let mut depth = 0usize; + 'table: loop { + for rule in state { + if match rule.0 { + Input::Keyword(expected) => input.step(|cursor| match cursor.ident() { + Some((ident, rest)) if ident == expected => Ok((true, rest)), + _ => Ok((false, *cursor)), + })?, + Input::Punct(expected) => input.step(|cursor| { + let begin = *cursor; + let mut cursor = begin; + for (i, ch) in expected.chars().enumerate() { + match cursor.punct() { + Some((punct, _)) if punct.as_char() != ch => break, + Some((_, rest)) if i == expected.len() - 1 => { + return Ok((true, rest)); + } + Some((punct, rest)) if punct.spacing() == Spacing::Joint => { + cursor = rest; + } + _ => break, + } + } + Ok((false, begin)) + })?, + Input::ConsumeAny => input.parse::>()?.is_some(), + Input::ConsumeBinOp => input.parse::().is_ok(), + Input::ConsumeBrace | Input::ConsumeNestedBrace => { + (matches!(rule.0, Input::ConsumeBrace) || depth > 0) + && input.step(|cursor| match cursor.group(Delimiter::Brace) { + Some((_inside, _span, rest)) => Ok((true, rest)), + None => Ok((false, *cursor)), + })? + } + Input::ConsumeDelimiter => input.step(|cursor| match cursor.any_group() { + Some((_inside, _delimiter, _span, rest)) => Ok((true, rest)), + None => Ok((false, *cursor)), + })?, + Input::ConsumeIdent => input.parse::>()?.is_some(), + Input::ConsumeLifetime => input.parse::>()?.is_some(), + Input::ConsumeLiteral => input.parse::>()?.is_some(), + Input::ExpectPath => { + input.parse::()?; + true + } + Input::ExpectTurbofish => { + if input.peek(Token![::]) { + input.parse::()?; + } + true + } + Input::ExpectType => { + Type::without_plus(input)?; + true + } + Input::CanBeginExpr => Expr::peek(input), + Input::Otherwise => true, + Input::Empty => input.is_empty() || input.peek(Token![,]), + } { + state = match rule.1 { + Action::SetState(next) => next, + Action::IncDepth => (depth += 1, &INIT).1, + Action::DecDepth => (depth -= 1, &POSTFIX).1, + Action::Finish => return if depth == 0 { Ok(()) } else { break }, + }; + continue 'table; + } + } + return Err(input.error("unsupported expression")); + } +} diff --git a/tests/test_precedence.rs b/tests/test_precedence.rs index 7231bd6409..1fa64b3205 100644 --- a/tests/test_precedence.rs +++ b/tests/test_precedence.rs @@ -49,6 +49,7 @@ use std::fs; use std::path::Path; use std::process; use std::sync::atomic::{AtomicUsize, Ordering}; +use syn::parse::Parser as _; #[macro_use] mod macros; @@ -56,6 +57,9 @@ mod macros; mod common; mod repo; +#[path = "../src/scan_expr.rs"] +mod scan_expr; + #[test] fn test_rustc_precedence() { repo::rayon_init(); @@ -115,7 +119,8 @@ fn test_expressions(path: &Path, edition: Edition, exprs: Vec) -> (us rustc_span::create_session_if_not_set_then(edition, |_| { for expr in exprs { - let source_code = expr.to_token_stream().to_string(); + let expr_tokens = expr.to_token_stream(); + let source_code = expr_tokens.to_string(); let librustc_ast = if let Some(e) = librustc_parse_and_rewrite(&source_code) { e } else { @@ -173,6 +178,16 @@ fn test_expressions(path: &Path, edition: Edition, exprs: Vec) -> (us continue; } + if scan_expr::scan_expr.parse2(expr_tokens).is_err() { + failed += 1; + errorf!( + "\nFAIL {} - failed to scan expr\n{}\n", + path.display(), + source_code, + ); + continue; + } + passed += 1; } });