Use speculative parsing for with-items

crates/ruff_python_parser/resources/inline/err/with_items_parenthesized_missing_colon.py

@@ -0,0 +1,3 @@
+# `)` followed by a newline
+with (item1, item2)
+    pass

crates/ruff_python_parser/src/parser/expression.rs

@@ -689,7 +689,8 @@ impl<'src> Parser<'src> {
 
                         parsed_expr = Expr::Generator(parser.parse_generator_expression(
                             parsed_expr.expr,
-                            GeneratorExpressionInParentheses::No(start),
+                            start,
+                            Parenthesized::No,
                         ))
                         .into();
                     }
@@ -1705,7 +1706,8 @@ impl<'src> Parser<'src> {
 
                 let generator = Expr::Generator(self.parse_generator_expression(
                     parsed_expr.expr,
-                    GeneratorExpressionInParentheses::Yes(start),
+                    start,
+                    Parenthesized::Yes,
                 ));
 
                 ParsedExpr {
@@ -1929,46 +1931,27 @@ impl<'src> Parser<'src> {
 
     /// Parses a generator expression.
     ///
-    /// The given `in_parentheses` parameter is used to determine whether the generator
-    /// expression is enclosed in parentheses or not:
-    /// - `Yes`, expect the `)` token after the generator expression.
-    /// - `No`, no parentheses are expected.
-    /// - `Maybe`, consume the `)` token if it's present.
-    ///
-    /// The contained start position in each variant is used to determine the range
-    /// of the generator expression.
+    /// The given `start` offset is the start of either the opening parenthesis if the generator is
+    /// parenthesized or the first token of the expression.
     ///
     /// See: <https://docs.python.org/3/reference/expressions.html#generator-expressions>
     pub(super) fn parse_generator_expression(
         &mut self,
         element: Expr,
-        in_parentheses: GeneratorExpressionInParentheses,
+        start: TextSize,
+        parenthesized: Parenthesized,
     ) -> ast::ExprGenerator {
         let generators = self.parse_generators();
 
-        let (parenthesized, start) = match in_parentheses {
-            GeneratorExpressionInParentheses::Yes(lpar_start) => {
-                self.expect(TokenKind::Rpar);
-                (true, lpar_start)
-            }
-            GeneratorExpressionInParentheses::No(expr_start) => (false, expr_start),
-            GeneratorExpressionInParentheses::Maybe {
-                lpar_start,
-                expr_start,
-            } => {
-                if self.eat(TokenKind::Rpar) {
-                    (true, lpar_start)
-                } else {
-                    (false, expr_start)
-                }
-            }
-        };
+        if parenthesized.is_yes() {
+            self.expect(TokenKind::Rpar);
+        }
 
         ast::ExprGenerator {
             elt: Box::new(element),
             generators,
             range: self.node_range(start),
-            parenthesized,
+            parenthesized: parenthesized.is_yes(),
         }
     }
 
@@ -2472,26 +2455,6 @@ impl From<Operator> for OperatorPrecedence {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(super) enum GeneratorExpressionInParentheses {
-    /// The generator expression is in parentheses. The given [`TextSize`] is the
-    /// start of the left parenthesis. E.g., `(x for x in range(10))`.
-    Yes(TextSize),
-
-    /// The generator expression is not in parentheses. The given [`TextSize`] is the
-    /// start of the expression. E.g., `x for x in range(10)`.
-    No(TextSize),
-
-    /// The generator expression may or may not be in parentheses. The given [`TextSize`]s
-    /// are the start of the left parenthesis and the start of the expression, respectively.
-    Maybe {
-        /// The start of the left parenthesis.
-        lpar_start: TextSize,
-        /// The start of the expression.
-        expr_start: TextSize,
-    },
-}
-
 /// Represents the precedence used for parsing the value part of a starred expression.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(super) enum StarredExpressionPrecedence {

crates/ruff_python_parser/src/parser/mod.rs

@@ -705,16 +705,6 @@ enum WithItemKind {
     /// The parentheses belongs to the context expression.
     ParenthesizedExpression,
 
-    /// A list of `with` items that has only one item which is a parenthesized
-    /// generator expression.
-    ///
-    /// ```python
-    /// with (x for x in range(10)): ...
-    /// ```
-    ///
-    /// The parentheses belongs to the generator expression.
-    SingleParenthesizedGeneratorExpression,
-
     /// The `with` items aren't parenthesized in any way.
     ///
     /// ```python
@@ -732,20 +722,15 @@ impl WithItemKind {
     const fn list_terminator(self) -> TokenKind {
         match self {
             WithItemKind::Parenthesized => TokenKind::Rpar,
-            WithItemKind::Unparenthesized
-            | WithItemKind::ParenthesizedExpression
-            | WithItemKind::SingleParenthesizedGeneratorExpression => TokenKind::Colon,
+            WithItemKind::Unparenthesized | WithItemKind::ParenthesizedExpression => {
+                TokenKind::Colon
+            }
         }
     }
 
-    /// Returns `true` if the `with` item is a parenthesized expression i.e., the
-    /// parentheses belong to the context expression.
-    const fn is_parenthesized_expression(self) -> bool {
-        matches!(
-            self,
-            WithItemKind::ParenthesizedExpression
-                | WithItemKind::SingleParenthesizedGeneratorExpression
-        )
+    /// Returns `true` if the with items are parenthesized.
+    const fn is_parenthesized(self) -> bool {
+        matches!(self, WithItemKind::Parenthesized)
     }
 }
 
@@ -1172,7 +1157,6 @@ bitflags! {
         const LAMBDA_PARAMETERS = 1 << 24;
         const WITH_ITEMS_PARENTHESIZED = 1 << 25;
         const WITH_ITEMS_PARENTHESIZED_EXPRESSION = 1 << 26;
-        const WITH_ITEMS_SINGLE_PARENTHESIZED_GENERATOR_EXPRESSION = 1 << 27;
         const WITH_ITEMS_UNPARENTHESIZED = 1 << 28;
         const F_STRING_ELEMENTS = 1 << 29;
     }
@@ -1225,9 +1209,6 @@ impl RecoveryContext {
                 WithItemKind::ParenthesizedExpression => {
                     RecoveryContext::WITH_ITEMS_PARENTHESIZED_EXPRESSION
                 }
-                WithItemKind::SingleParenthesizedGeneratorExpression => {
-                    RecoveryContext::WITH_ITEMS_SINGLE_PARENTHESIZED_GENERATOR_EXPRESSION
-                }
                 WithItemKind::Unparenthesized => RecoveryContext::WITH_ITEMS_UNPARENTHESIZED,
             },
             RecoveryContextKind::FStringElements => RecoveryContext::F_STRING_ELEMENTS,
@@ -1294,9 +1275,6 @@ impl RecoveryContext {
             RecoveryContext::WITH_ITEMS_PARENTHESIZED_EXPRESSION => {
                 RecoveryContextKind::WithItems(WithItemKind::ParenthesizedExpression)
             }
-            RecoveryContext::WITH_ITEMS_SINGLE_PARENTHESIZED_GENERATOR_EXPRESSION => {
-                RecoveryContextKind::WithItems(WithItemKind::SingleParenthesizedGeneratorExpression)
-            }
             RecoveryContext::WITH_ITEMS_UNPARENTHESIZED => {
                 RecoveryContextKind::WithItems(WithItemKind::Unparenthesized)
             }