Replace autogenerated parser with hand-written parser

crates/ruff_linter/src/logging.rs

@@ -173,7 +173,7 @@ impl DisplayParseError {
         // Translate the byte offset to a location in the originating source.
         let location =
             if let Some(jupyter_index) = source_kind.as_ipy_notebook().map(Notebook::index) {
-                let source_location = source_code.source_location(error.offset);
+                let source_location = source_code.source_location(error.location.start());
 
                 ErrorLocation::Cell(
                     jupyter_index
@@ -187,7 +187,7 @@ impl DisplayParseError {
                     },
                 )
             } else {
-                ErrorLocation::File(source_code.source_location(error.offset))
+                ErrorLocation::File(source_code.source_location(error.location.start()))
             };
 
         Self {
@@ -254,27 +254,7 @@ impl<'a> DisplayParseErrorType<'a> {
 
 impl Display for DisplayParseErrorType<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        match self.0 {
-            ParseErrorType::Eof => write!(f, "Expected token but reached end of file."),
-            ParseErrorType::ExtraToken(ref tok) => write!(
-                f,
-                "Got extraneous token: {tok}",
-                tok = TruncateAtNewline(&tok)
-            ),
-            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
-            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
-                if let Some(expected) = expected.as_ref() {
-                    write!(
-                        f,
-                        "Expected '{expected}', but got {tok}",
-                        tok = TruncateAtNewline(&tok)
-                    )
-                } else {
-                    write!(f, "Unexpected token {tok}", tok = TruncateAtNewline(&tok))
-                }
-            }
-            ParseErrorType::Lexical(ref error) => write!(f, "{error}"),
-        }
+        write!(f, "{}", TruncateAtNewline(&self.0))
     }
 }
 

crates/ruff_linter/src/rules/flake8_pytest_style/rules/helpers.rs

@@ -74,6 +74,7 @@ fn is_empty_or_null_fstring_element(element: &ast::FStringElement) -> bool {
         ast::FStringElement::Expression(ast::FStringExpressionElement { expression, .. }) => {
             is_empty_or_null_string(expression)
         }
+        ast::FStringElement::Invalid(_) => false,
     }
 }
 

crates/ruff_linter/src/rules/pycodestyle/rules/errors.rs

@@ -81,7 +81,7 @@ pub(crate) fn syntax_error(
     parse_error: &ParseError,
     locator: &Locator,
 ) {
-    let rest = locator.after(parse_error.offset);
+    let rest = locator.after(parse_error.location.start());
 
     // Try to create a non-empty range so that the diagnostic can print a caret at the
     // right position. This requires that we retrieve the next character, if any, and take its length
@@ -95,6 +95,6 @@ pub(crate) fn syntax_error(
         SyntaxError {
             message: format!("{}", DisplayParseErrorType::new(&parse_error.error)),
         },
-        TextRange::at(parse_error.offset, len),
+        TextRange::at(parse_error.location.start(), len),
     ));
 }

crates/ruff_linter/src/rules/pylint/rules/assert_on_string_literal.rs

@@ -77,7 +77,9 @@ pub(crate) fn assert_on_string_literal(checker: &mut Checker, test: &Expr) {
                         ast::FStringElement::Literal(ast::FStringLiteralElement {
                             value, ..
                         }) => value.is_empty(),
-                        ast::FStringElement::Expression(_) => false,
+                        ast::FStringElement::Expression(_) | ast::FStringElement::Invalid(_) => {
+                            false
+                        }
                     })
                 }
             }) {
@@ -89,7 +91,9 @@ pub(crate) fn assert_on_string_literal(checker: &mut Checker, test: &Expr) {
                         ast::FStringElement::Literal(ast::FStringLiteralElement {
                             value, ..
                         }) => !value.is_empty(),
-                        ast::FStringElement::Expression(_) => false,
+                        ast::FStringElement::Expression(_) | ast::FStringElement::Invalid(_) => {
+                            false
+                        }
                     })
                 }
             }) {

crates/ruff_linter/src/rules/pyupgrade/rules/use_pep604_annotation.rs

@@ -203,6 +203,7 @@ fn is_allowed_value(expr: &Expr) -> bool {
         | Expr::YieldFrom(_)
         | Expr::Starred(_)
         | Expr::Slice(_)
-        | Expr::IpyEscapeCommand(_) => false,
+        | Expr::IpyEscapeCommand(_)
+        | Expr::Invalid(_) => false,
     }
 }

crates/ruff_linter/src/rules/refurb/rules/bit_count.rs

@@ -159,7 +159,8 @@ pub(crate) fn bit_count(checker: &mut Checker, call: &ExprCall) {
         | Expr::NoneLiteral(_)
         | Expr::EllipsisLiteral(_)
         | Expr::Attribute(_)
-        | Expr::Subscript(_) => false,
+        | Expr::Subscript(_)
+        | Expr::Invalid(_) => false,
     };
 
     let replacement = if parenthesize {

crates/ruff_linter/src/rules/ruff/rules/unreachable.rs

@@ -439,7 +439,8 @@ fn is_wildcard(pattern: &MatchCase) -> bool {
             | Pattern::MatchSequence(_)
             | Pattern::MatchMapping(_)
             | Pattern::MatchClass(_)
-            | Pattern::MatchStar(_) => false,
+            | Pattern::MatchStar(_)
+            | Pattern::Invalid(_) => false,
             Pattern::MatchAs(PatternMatchAs { pattern, .. }) => pattern.is_none(),
             Pattern::MatchOr(PatternMatchOr { patterns, .. }) => {
                 patterns.iter().all(is_wildcard_pattern)
@@ -648,6 +649,8 @@ impl<'stmt> BasicBlocksBuilder<'stmt> {
                         | Expr::Name(_)
                         | Expr::List(_)
                         | Expr::IpyEscapeCommand(_)
+                        // NOTE: Is this the correct place to handle this node?
+                        | Expr::Invalid(_)
                         | Expr::Tuple(_)
                         | Expr::Slice(_) => self.unconditional_next_block(after),
                         // TODO: handle these expressions.

crates/ruff_python_ast/src/comparable.rs

@@ -234,6 +234,7 @@ pub enum ComparablePattern<'a> {
     MatchStar(PatternMatchStar<'a>),
     MatchAs(PatternMatchAs<'a>),
     MatchOr(PatternMatchOr<'a>),
+    Invalid,
 }
 
 impl<'a> From<&'a ast::Pattern> for ComparablePattern<'a> {
@@ -286,6 +287,7 @@ impl<'a> From<&'a ast::Pattern> for ComparablePattern<'a> {
                     patterns: patterns.iter().map(Into::into).collect(),
                 })
             }
+            ast::Pattern::Invalid(_) => Self::Invalid,
         }
     }
 }
@@ -513,6 +515,7 @@ impl<'a> From<&'a ast::ExceptHandler> for ComparableExceptHandler<'a> {
 pub enum ComparableFStringElement<'a> {
     Literal(&'a str),
     FStringExpressionElement(FStringExpressionElement<'a>),
+    Invalid,
 }
 
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -540,6 +543,7 @@ impl<'a> From<&'a ast::FStringElement> for ComparableFStringElement<'a> {
                         .map(|spec| spec.elements.iter().map(Into::into).collect()),
                 })
             }
+            ast::FStringElement::Invalid(_) => Self::Invalid,
         }
     }
 }
@@ -864,6 +868,7 @@ pub enum ComparableExpr<'a> {
     Tuple(ExprTuple<'a>),
     Slice(ExprSlice<'a>),
     IpyEscapeCommand(ExprIpyEscapeCommand<'a>),
+    Invalid,
 }
 
 impl<'a> From<&'a Box<ast::Expr>> for Box<ComparableExpr<'a>> {
@@ -1093,6 +1098,7 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> {
                 kind: *kind,
                 value: value.as_str(),
             }),
+            ast::Expr::Invalid(_) => Self::Invalid,
         }
     }
 }

crates/ruff_python_ast/src/expression.rs

@@ -38,6 +38,7 @@ pub enum ExpressionRef<'a> {
     Tuple(&'a ast::ExprTuple),
     Slice(&'a ast::ExprSlice),
     IpyEscapeCommand(&'a ast::ExprIpyEscapeCommand),
+    Invalid(&'a ast::ExprInvalid),
 }
 
 impl<'a> From<&'a Box<Expr>> for ExpressionRef<'a> {
@@ -81,6 +82,7 @@ impl<'a> From<&'a Expr> for ExpressionRef<'a> {
             Expr::Tuple(value) => ExpressionRef::Tuple(value),
             Expr::Slice(value) => ExpressionRef::Slice(value),
             Expr::IpyEscapeCommand(value) => ExpressionRef::IpyEscapeCommand(value),
+            Expr::Invalid(value) => ExpressionRef::Invalid(value),
         }
     }
 }
@@ -285,6 +287,7 @@ impl<'a> From<ExpressionRef<'a>> for AnyNodeRef<'a> {
             ExpressionRef::IpyEscapeCommand(expression) => {
                 AnyNodeRef::ExprIpyEscapeCommand(expression)
             }
+            ExpressionRef::Invalid(expression) => AnyNodeRef::ExprInvalid(expression),
         }
     }
 }
@@ -324,6 +327,7 @@ impl Ranged for ExpressionRef<'_> {
             ExpressionRef::Tuple(expression) => expression.range(),
             ExpressionRef::Slice(expression) => expression.range(),
             ExpressionRef::IpyEscapeCommand(expression) => expression.range(),
+            ExpressionRef::Invalid(expression) => expression.range(),
         }
     }
 }

crates/ruff_python_ast/src/helpers.rs

@@ -258,7 +258,8 @@ pub fn any_over_expr(expr: &Expr, func: &dyn Fn(&Expr) -> bool) -> bool {
         | Expr::BooleanLiteral(_)
         | Expr::NoneLiteral(_)
         | Expr::EllipsisLiteral(_)
-        | Expr::IpyEscapeCommand(_) => false,
+        | Expr::IpyEscapeCommand(_)
+        | Expr::Invalid(_) => false,
     }
 }
 
@@ -277,7 +278,7 @@ pub fn any_over_pattern(pattern: &Pattern, func: &dyn Fn(&Expr) -> bool) -> bool
         Pattern::MatchValue(ast::PatternMatchValue { value, range: _ }) => {
             any_over_expr(value, func)
         }
-        Pattern::MatchSingleton(_) => false,
+        Pattern::MatchSingleton(_) | Pattern::Invalid(_) => false,
         Pattern::MatchSequence(ast::PatternMatchSequence { patterns, range: _ }) => patterns
             .iter()
             .any(|pattern| any_over_pattern(pattern, func)),
@@ -310,7 +311,7 @@ pub fn any_over_pattern(pattern: &Pattern, func: &dyn Fn(&Expr) -> bool) -> bool
 
 pub fn any_over_f_string_element(element: &FStringElement, func: &dyn Fn(&Expr) -> bool) -> bool {
     match element {
-        FStringElement::Literal(_) => false,
+        FStringElement::Literal(_) | FStringElement::Invalid(_) => false,
         FStringElement::Expression(ast::FStringExpressionElement {
             expression,
             format_spec,
@@ -1184,6 +1185,7 @@ impl Truthiness {
                             value, ..
                         }) => !value.is_empty(),
                         ast::FStringElement::Expression(_) => true,
+                        ast::FStringElement::Invalid(_) => false,
                     })
                 {
                     Self::Truthy
@@ -1427,7 +1429,7 @@ mod tests {
     fn any_over_stmt_type_alias() {
         let seen = RefCell::new(Vec::new());
         let name = Expr::Name(ExprName {
-            id: "x".to_string(),
+            id: "x".into(),
             range: TextRange::default(),
             ctx: ExprContext::Load,
         });