Skip to content

Commit

Permalink
Remove leftover marker tokens (#11444)
Browse files Browse the repository at this point in the history
## Summary

This PR removes the leftover marker tokens from the LALRPOP to
hand-written parser migration.
  • Loading branch information
dhruvmanila authored May 16, 2024
1 parent 4436dec commit f67c02c
Showing 1 changed file with 7 additions and 25 deletions.
32 changes: 7 additions & 25 deletions crates/ruff_python_parser/src/token.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@
//!
//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Grammar/Tokens

use ruff_python_ast::{AnyStringFlags, BoolOp, Int, IpyEscapeKind, Operator, UnaryOp};
use std::fmt;

use crate::Mode;
use ruff_python_ast::{AnyStringFlags, BoolOp, Int, IpyEscapeKind, Operator, UnaryOp};

/// The set of tokens the Python source code can be tokenized in.
#[derive(Clone, Debug, PartialEq, is_macro::Is)]
Expand Down Expand Up @@ -61,7 +60,9 @@ pub enum Tok {
/// Token value for the end of an f-string. This includes the closing quote.
FStringEnd,
/// Token value for IPython escape commands. These are recognized by the lexer
/// only when the mode is [`Mode::Ipython`].
/// only when the mode is [`Ipython`].
///
/// [`Ipython`]: crate::Mode::Ipython
IpyEscapeCommand {
/// The magic command value.
value: Box<str>,
Expand All @@ -80,7 +81,9 @@ pub enum Tok {
/// Token value for a dedent.
Dedent,
EndOfFile,
/// Token value for a question mark `?`. This is only used in [`Mode::Ipython`].
/// Token value for a question mark `?`. This is only used in [`Ipython`].
///
/// [`Ipython`]: crate::Mode::Ipython
Question,
/// Token value for a exclamation mark `!`.
Exclamation,
Expand Down Expand Up @@ -222,23 +225,13 @@ pub enum Tok {
Yield,

Unknown,
// RustPython specific.
StartModule,
StartExpression,
}

impl Tok {
#[inline]
pub fn kind(&self) -> TokenKind {
TokenKind::from_token(self)
}

pub fn start_marker(mode: Mode) -> Self {
match mode {
Mode::Module | Mode::Ipython => Tok::StartModule,
Mode::Expression => Tok::StartExpression,
}
}
}

impl fmt::Display for Tok {
Expand All @@ -261,8 +254,6 @@ impl fmt::Display for Tok {
NonLogicalNewline => f.write_str("NonLogicalNewline"),
Indent => f.write_str("Indent"),
Dedent => f.write_str("Dedent"),
StartModule => f.write_str("StartProgram"),
StartExpression => f.write_str("StartExpression"),
EndOfFile => f.write_str("EOF"),
Question => f.write_str("?"),
Exclamation => f.write_str("!"),
Expand Down Expand Up @@ -537,10 +528,6 @@ pub enum TokenKind {
Yield,

Unknown,
// RustPython specific.
StartModule,
StartInteractive,
StartExpression,
}

impl TokenKind {
Expand Down Expand Up @@ -900,8 +887,6 @@ impl TokenKind {
Tok::With => TokenKind::With,
Tok::Yield => TokenKind::Yield,
Tok::Unknown => TokenKind::Unknown,
Tok::StartModule => TokenKind::StartModule,
Tok::StartExpression => TokenKind::StartExpression,
}
}
}
Expand Down Expand Up @@ -965,9 +950,6 @@ impl fmt::Display for TokenKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let value = match self {
TokenKind::Unknown => "Unknown",
TokenKind::StartModule => "StartModule",
TokenKind::StartExpression => "StartExpression",
TokenKind::StartInteractive => "StartInteractive",
TokenKind::Newline => "newline",
TokenKind::NonLogicalNewline => "NonLogicalNewline",
TokenKind::Indent => "indent",
Expand Down

0 comments on commit f67c02c

Please sign in to comment.