diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_after_keyword.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_after_keyword.rs index 296d9514bda6e9..d8dc7762c748a7 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_after_keyword.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_after_keyword.rs @@ -52,7 +52,7 @@ pub(crate) fn missing_whitespace_after_keyword( let tok0_kind = tok0.kind(); let tok1_kind = tok1.kind(); - if tok0_kind.is_keyword() + if tok0_kind.is_non_soft_keyword() && !(tok0_kind.is_singleton() || matches!(tok0_kind, TokenKind::Async | TokenKind::Await) || tok0_kind == TokenKind::Except && tok1_kind == TokenKind::Star diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_around_operator.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_around_operator.rs index 0015c3ac26db76..ba1c3712fdd0b4 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_around_operator.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_around_operator.rs @@ -198,9 +198,7 @@ pub(crate) fn missing_whitespace_around_operator( matches!( prev_kind, TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace - ) || !(prev_kind.is_operator() - || prev_kind.is_keyword() - || prev_kind.is_soft_keyword()) + ) || !(prev_kind.is_operator() || prev_kind.is_keyword()) }; if is_binary { diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs index 606972bcf0c38b..933fe3bbfd3cdf 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs @@ -445,7 +445,7 @@ impl LogicalLinesBuilder { if matches!(kind, TokenKind::Comma | TokenKind::Semi | TokenKind::Colon) { line.flags.insert(TokenFlags::PUNCTUATION); - } else if kind.is_keyword() { + } else if kind.is_non_soft_keyword() { line.flags.insert(TokenFlags::KEYWORD); } diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/whitespace_around_keywords.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/whitespace_around_keywords.rs index 365060f41d0e5e..6914a386cb0830 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/whitespace_around_keywords.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/whitespace_around_keywords.rs @@ -127,8 +127,8 @@ pub(crate) fn whitespace_around_keywords(line: &LogicalLine, context: &mut Logic let mut after_keyword = false; for token in line.tokens() { - let is_keyword = token.kind().is_keyword(); - if is_keyword { + let is_non_soft_keyword = token.kind().is_non_soft_keyword(); + if is_non_soft_keyword { if !after_keyword { match line.leading_whitespace(token) { (Whitespace::Tab, offset) => { @@ -184,6 +184,6 @@ pub(crate) fn whitespace_around_keywords(line: &LogicalLine, context: &mut Logic } } - after_keyword = is_keyword; + after_keyword = is_non_soft_keyword; } } diff --git a/crates/ruff_python_parser/src/token.rs b/crates/ruff_python_parser/src/token.rs index 1e3af0e5be2638..76016de4fc1986 100644 --- a/crates/ruff_python_parser/src/token.rs +++ b/crates/ruff_python_parser/src/token.rs @@ -352,7 +352,7 @@ impl fmt::Display for Tok { /// /// This is a lightweight representation of [`Tok`] which doesn't contain any information /// about the token itself. -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] pub enum TokenKind { /// Token value for a name, commonly known as an identifier. Name, @@ -485,12 +485,10 @@ pub enum TokenKind { /// Token value for ellipsis `...`. Ellipsis, - // Self documenting. - // Keywords (alphabetically): - False, - None, - True, + // The keywords should be sorted in alphabetical order. If the boundary tokens for the + // "Keywords" and "Soft keywords" group change, update the related methods on `TokenKind`. + // Keywords And, As, Assert, @@ -504,6 +502,7 @@ pub enum TokenKind { Elif, Else, Except, + False, Finally, For, From, @@ -513,20 +512,24 @@ pub enum TokenKind { In, Is, Lambda, + None, Nonlocal, Not, Or, Pass, Raise, Return, + True, Try, While, - Match, - Type, - Case, With, Yield, + // Soft keywords + Case, + Match, + Type, + Unknown, } @@ -536,45 +539,28 @@ impl TokenKind { matches!(self, TokenKind::Newline | TokenKind::NonLogicalNewline) } + /// Returns `true` if the token is a keyword (including soft keywords). + /// + /// See also [`TokenKind::is_soft_keyword`], [`TokenKind::is_non_soft_keyword`]. #[inline] - pub const fn is_keyword(self) -> bool { - matches!( - self, - TokenKind::False - | TokenKind::True - | TokenKind::None - | TokenKind::And - | TokenKind::As - | TokenKind::Assert - | TokenKind::Await - | TokenKind::Break - | TokenKind::Class - | TokenKind::Continue - | TokenKind::Def - | TokenKind::Del - | TokenKind::Elif - | TokenKind::Else - | TokenKind::Except - | TokenKind::Finally - | TokenKind::For - | TokenKind::From - | TokenKind::Global - | TokenKind::If - | TokenKind::Import - | TokenKind::In - | TokenKind::Is - | TokenKind::Lambda - | TokenKind::Nonlocal - | TokenKind::Not - | TokenKind::Or - | TokenKind::Pass - | TokenKind::Raise - | TokenKind::Return - | TokenKind::Try - | TokenKind::While - | TokenKind::With - | TokenKind::Yield - ) + pub fn is_keyword(self) -> bool { + TokenKind::And <= self && self <= TokenKind::Type + } + + /// Returns `true` if the token is strictly a soft keyword. + /// + /// See also [`TokenKind::is_keyword`], [`TokenKind::is_non_soft_keyword`]. + #[inline] + pub fn is_soft_keyword(self) -> bool { + TokenKind::Case <= self && self <= TokenKind::Type + } + + /// Returns `true` if the token is strictly a non-soft keyword. + /// + /// See also [`TokenKind::is_keyword`], [`TokenKind::is_soft_keyword`]. + #[inline] + pub fn is_non_soft_keyword(self) -> bool { + TokenKind::And <= self && self <= TokenKind::Yield } #[inline] @@ -685,11 +671,6 @@ impl TokenKind { ) } - #[inline] - pub const fn is_soft_keyword(self) -> bool { - matches!(self, TokenKind::Match | TokenKind::Case) - } - /// Returns `true` if the current token is a unary arithmetic operator. #[inline] pub const fn is_unary_arithmetic_operator(self) -> bool { diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ann_assign_stmt_type_alias_annotation.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ann_assign_stmt_type_alias_annotation.py.snap index 43d7ecb18673ba..3ced503debc870 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ann_assign_stmt_type_alias_annotation.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ann_assign_stmt_type_alias_annotation.py.snap @@ -11,7 +11,7 @@ Module( body: [ AnnAssign( StmtAnnAssign { - range: 0..2, + range: 0..7, target: Name( ExprName { range: 0..1, @@ -21,26 +21,27 @@ Module( ), annotation: Name( ExprName { - range: 2..2, - id: "", - ctx: Invalid, + range: 3..7, + id: "type", + ctx: Load, }, ), value: None, simple: true, }, ), - TypeAlias( - StmtTypeAlias { - range: 3..15, - name: Name( - ExprName { - range: 8..9, - id: "X", - ctx: Store, - }, - ), - type_params: None, + Assign( + StmtAssign { + range: 8..15, + targets: [ + Name( + ExprName { + range: 8..9, + id: "X", + ctx: Store, + }, + ), + ], value: Name( ExprName { range: 12..15, @@ -52,33 +53,34 @@ Module( ), Expr( StmtExpr { - range: 16..23, + range: 16..28, value: Lambda( ExprLambda { - range: 16..23, + range: 16..28, parameters: None, body: Name( ExprName { - range: 23..23, - id: "", - ctx: Invalid, + range: 24..28, + id: "type", + ctx: Load, }, ), }, ), }, ), - TypeAlias( - StmtTypeAlias { - range: 24..36, - name: Name( - ExprName { - range: 29..30, - id: "X", - ctx: Store, - }, - ), - type_params: None, + Assign( + StmtAssign { + range: 29..36, + targets: [ + Name( + ExprName { + range: 29..30, + id: "X", + ctx: Store, + }, + ), + ], value: Name( ExprName { range: 33..36, @@ -96,13 +98,27 @@ Module( | 1 | a: type X = int - | ^^^^ Syntax Error: Expected an expression + | ^^^^ Syntax Error: Expected an identifier, but found a keyword 'type' that cannot be used here +2 | lambda: type X = int + | + + + | +1 | a: type X = int + | ^ Syntax Error: Simple statements must be separated by newlines or semicolons +2 | lambda: type X = int + | + + + | +1 | a: type X = int 2 | lambda: type X = int + | ^^^^ Syntax Error: Expected an identifier, but found a keyword 'type' that cannot be used here | | 1 | a: type X = int 2 | lambda: type X = int - | ^^^^ Syntax Error: Expected an expression + | ^ Syntax Error: Simple statements must be separated by newlines or semicolons | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@comprehension_missing_for_after_async.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@comprehension_missing_for_after_async.py.snap index 4297ad0f41fee2..e506dac043a4f2 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@comprehension_missing_for_after_async.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@comprehension_missing_for_after_async.py.snap @@ -12,38 +12,11 @@ Module( Expr( StmtExpr { range: 0..7, - value: Generator( - ExprGenerator { - range: 0..7, - elt: Name( - ExprName { - range: 1..1, - id: "", - ctx: Invalid, - }, - ), - generators: [ - Comprehension { - range: 1..6, - target: Name( - ExprName { - range: 6..6, - id: "", - ctx: Store, - }, - ), - iter: Name( - ExprName { - range: 6..6, - id: "", - ctx: Invalid, - }, - ), - ifs: [], - is_async: true, - }, - ], - parenthesized: true, + value: Name( + ExprName { + range: 1..6, + id: "async", + ctx: Load, }, ), }, @@ -95,14 +68,7 @@ Module( | 1 | (async) - | ^^^^^ Syntax Error: Expected an expression -2 | (x async x in iter) - | - - - | -1 | (async) - | ^ Syntax Error: Expected 'for', found ')' + | ^^^^^ Syntax Error: Expected an identifier, but found a keyword 'async' that cannot be used here 2 | (x async x in iter) |