Use speculative parsing for with-items

astral-sh · Jun 5, 2024 · 992da43 · 992da43
1 parent 2e0a975
commit 992da43
Show file tree

Hide file tree

Showing 7 changed files with 444 additions and 676 deletions.
diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs
@@ -689,7 +689,8 @@ impl<'src> Parser<'src> {
 
                         parsed_expr = Expr::Generator(parser.parse_generator_expression(
                             parsed_expr.expr,
-                            GeneratorExpressionInParentheses::No(start),
+                            start,
+                            Parenthesized::No,
                         ))
                         .into();
                     }
@@ -1705,7 +1706,8 @@ impl<'src> Parser<'src> {
 
                 let generator = Expr::Generator(self.parse_generator_expression(
                     parsed_expr.expr,
-                    GeneratorExpressionInParentheses::Yes(start),
+                    start,
+                    Parenthesized::Yes,
                 ));
 
                 ParsedExpr {
@@ -1942,33 +1944,20 @@ impl<'src> Parser<'src> {
     pub(super) fn parse_generator_expression(
         &mut self,
         element: Expr,
-        in_parentheses: GeneratorExpressionInParentheses,
+        start: TextSize,
+        parenthesized: Parenthesized,
     ) -> ast::ExprGenerator {
         let generators = self.parse_generators();
 
-        let (parenthesized, start) = match in_parentheses {
-            GeneratorExpressionInParentheses::Yes(lpar_start) => {
-                self.expect(TokenKind::Rpar);
-                (true, lpar_start)
-            }
-            GeneratorExpressionInParentheses::No(expr_start) => (false, expr_start),
-            GeneratorExpressionInParentheses::Maybe {
-                lpar_start,
-                expr_start,
-            } => {
-                if self.eat(TokenKind::Rpar) {
-                    (true, lpar_start)
-                } else {
-                    (false, expr_start)
-                }
-            }
-        };
+        if parenthesized.is_yes() {
+            self.expect(TokenKind::Rpar);
+        }
 
         ast::ExprGenerator {
             elt: Box::new(element),
             generators,
             range: self.node_range(start),
-            parenthesized,
+            parenthesized: parenthesized.is_yes(),
         }
     }
 
@@ -2472,26 +2461,6 @@ impl From<Operator> for OperatorPrecedence {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(super) enum GeneratorExpressionInParentheses {
-    /// The generator expression is in parentheses. The given [`TextSize`] is the
-    /// start of the left parenthesis. E.g., `(x for x in range(10))`.
-    Yes(TextSize),
-
-    /// The generator expression is not in parentheses. The given [`TextSize`] is the
-    /// start of the expression. E.g., `x for x in range(10)`.
-    No(TextSize),
-
-    /// The generator expression may or may not be in parentheses. The given [`TextSize`]s
-    /// are the start of the left parenthesis and the start of the expression, respectively.
-    Maybe {
-        /// The start of the left parenthesis.
-        lpar_start: TextSize,
-        /// The start of the expression.
-        expr_start: TextSize,
-    },
-}
-
 /// Represents the precedence used for parsing the value part of a starred expression.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(super) enum StarredExpressionPrecedence {

diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs
@@ -705,16 +705,6 @@ enum WithItemKind {
     /// The parentheses belongs to the context expression.
     ParenthesizedExpression,
 
-    /// A list of `with` items that has only one item which is a parenthesized
-    /// generator expression.
-    ///
-    /// ```python
-    /// with (x for x in range(10)): ...
-    /// ```
-    ///
-    /// The parentheses belongs to the generator expression.
-    SingleParenthesizedGeneratorExpression,
-
     /// The `with` items aren't parenthesized in any way.
     ///
     /// ```python
@@ -732,20 +722,15 @@ impl WithItemKind {
     const fn list_terminator(self) -> TokenKind {
         match self {
             WithItemKind::Parenthesized => TokenKind::Rpar,
-            WithItemKind::Unparenthesized
-            | WithItemKind::ParenthesizedExpression
-            | WithItemKind::SingleParenthesizedGeneratorExpression => TokenKind::Colon,
+            WithItemKind::Unparenthesized | WithItemKind::ParenthesizedExpression => {
+                TokenKind::Colon
+            }
         }
     }
 
-    /// Returns `true` if the `with` item is a parenthesized expression i.e., the
-    /// parentheses belong to the context expression.
-    const fn is_parenthesized_expression(self) -> bool {
-        matches!(
-            self,
-            WithItemKind::ParenthesizedExpression
-                | WithItemKind::SingleParenthesizedGeneratorExpression
-        )
+    /// Returns `true` if the with items are parenthesized.
+    const fn is_parenthesized(self) -> bool {
+        matches!(self, WithItemKind::Parenthesized)
     }
 }
 
@@ -1172,7 +1157,6 @@ bitflags! {
         const LAMBDA_PARAMETERS = 1 << 24;
         const WITH_ITEMS_PARENTHESIZED = 1 << 25;
         const WITH_ITEMS_PARENTHESIZED_EXPRESSION = 1 << 26;
-        const WITH_ITEMS_SINGLE_PARENTHESIZED_GENERATOR_EXPRESSION = 1 << 27;
         const WITH_ITEMS_UNPARENTHESIZED = 1 << 28;
         const F_STRING_ELEMENTS = 1 << 29;
     }
@@ -1225,9 +1209,6 @@ impl RecoveryContext {
                 WithItemKind::ParenthesizedExpression => {
                     RecoveryContext::WITH_ITEMS_PARENTHESIZED_EXPRESSION
                 }
-                WithItemKind::SingleParenthesizedGeneratorExpression => {
-                    RecoveryContext::WITH_ITEMS_SINGLE_PARENTHESIZED_GENERATOR_EXPRESSION
-                }
                 WithItemKind::Unparenthesized => RecoveryContext::WITH_ITEMS_UNPARENTHESIZED,
             },
             RecoveryContextKind::FStringElements => RecoveryContext::F_STRING_ELEMENTS,
@@ -1294,9 +1275,6 @@ impl RecoveryContext {
             RecoveryContext::WITH_ITEMS_PARENTHESIZED_EXPRESSION => {
                 RecoveryContextKind::WithItems(WithItemKind::ParenthesizedExpression)
             }
-            RecoveryContext::WITH_ITEMS_SINGLE_PARENTHESIZED_GENERATOR_EXPRESSION => {
-                RecoveryContextKind::WithItems(WithItemKind::SingleParenthesizedGeneratorExpression)
-            }
             RecoveryContext::WITH_ITEMS_UNPARENTHESIZED => {
                 RecoveryContextKind::WithItems(WithItemKind::Unparenthesized)
             }