Skip to content

Commit

Permalink
Do not include newline for unterminated string range (#12017)
Browse files Browse the repository at this point in the history 
## Summary

This PR updates the unterminated string error range to not include the
final newline character.

This is a follow-up to #12016 and required for #12019

This is not done for when the unterminated string goes till the end of
file (not a newline character). The unterminated f-string range is
correct.

### Why is this required for #12019 ?

Because otherwise the token ranges will overlap. For example:
```py
f"{"
f"{foo!r"
```

Here, the re-lexing logic recovers from an unterminated f-string and
thus emitting a `Newline` token for the one at the end of the first
line. But, currently the `Unknown` and the `Newline` token would overlap
because the `Unknown` token (unterminated string literal) range would
include the newline character.

## Test Plan

Update and validate the snapshot.
  • Loading branch information
@dhruvmanila
dhruvmanila authored Jun 25, 2024
1 parent 9c1b6ec commit d930e97
Show file tree
Hide file tree
Showing 5 changed files with 48 additions and 60 deletions.
4 changes: 2 additions & 2 deletions crates/ruff_python_parser/src/lexer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -973,10 +973,10 @@ impl<'src> Lexer<'src> {
}

match ch {
Some('\r' | '\n') => {
Some(newline @ ('\r' | '\n')) => {
return self.push_error(LexicalError::new(
LexicalErrorType::UnclosedStringError,
self.token_range(),
self.token_range().sub_end(newline.text_len()),
));
}
Some(ch) if ch == quote => {
Expand Down
68 changes: 31 additions & 37 deletions crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace.py.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ Module(
body: [
Expr(
StmtExpr {
range: 0..5,
range: 0..4,
value: FString(
ExprFString {
range: 0..5,
range: 0..4,
value: FStringValue {
inner: Single(
FString(
FString {
range: 0..5,
range: 0..4,
elements: [
Expression(
FStringExpressionElement {
Expand Down Expand Up @@ -52,19 +52,19 @@ Module(
),
Expr(
StmtExpr {
range: 5..15,
range: 5..14,
value: FString(
ExprFString {
range: 5..15,
range: 5..14,
value: FStringValue {
inner: Single(
FString(
FString {
range: 5..15,
range: 5..14,
elements: [
Expression(
FStringExpressionElement {
range: 7..15,
range: 7..14,
expression: Name(
ExprName {
range: 8..11,
Expand Down Expand Up @@ -93,15 +93,15 @@ Module(
),
Expr(
StmtExpr {
range: 15..24,
range: 15..23,
value: FString(
ExprFString {
range: 15..24,
range: 15..23,
value: FStringValue {
inner: Single(
FString(
FString {
range: 15..24,
range: 15..23,
elements: [
Expression(
FStringExpressionElement {
Expand Down Expand Up @@ -148,7 +148,7 @@ Module(
[
FString(
FString {
range: 24..29,
range: 24..28,
elements: [
Expression(
FStringExpressionElement {
Expand Down Expand Up @@ -214,12 +214,10 @@ Module(
## Errors

|
1 | f"{"
| ____^
2 | | f"{foo!r"
| |_^ Syntax Error: missing closing quote in string literal
3 | f"{foo="
4 | f"{"
1 | f"{"
| ^ Syntax Error: missing closing quote in string literal
2 | f"{foo!r"
3 | f"{foo="
|


Expand All @@ -240,13 +238,11 @@ Module(


|
1 | f"{"
2 | f"{foo!r"
| ________^
3 | | f"{foo="
| |_^ Syntax Error: missing closing quote in string literal
4 | f"{"
5 | f"""{"""
1 | f"{"
2 | f"{foo!r"
| ^^ Syntax Error: missing closing quote in string literal
3 | f"{foo="
4 | f"{"
|


Expand Down Expand Up @@ -288,13 +284,12 @@ Module(


|
1 | f"{"
2 | f"{foo!r"
3 | f"{foo="
| ________^
4 | | f"{"
| |_^ Syntax Error: missing closing quote in string literal
5 | f"""{"""
1 | f"{"
2 | f"{foo!r"
3 | f"{foo="
| ^ Syntax Error: missing closing quote in string literal
4 | f"{"
5 | f"""{"""
|


Expand All @@ -319,12 +314,11 @@ Module(


|
2 | f"{foo!r"
3 | f"{foo="
4 | f"{"
| ____^
5 | | f"""{"""
| |_^ Syntax Error: missing closing quote in string literal
2 | f"{foo!r"
3 | f"{foo="
4 | f"{"
| ^ Syntax Error: missing closing quote in string literal
5 | f"""{"""
|


Expand Down
10 changes: 4 additions & 6 deletions ...parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string.py.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,10 @@ Module(
## Errors

|
1 | 'hello' 'world
| _________^
2 | | 1 + 1
| |_^ Syntax Error: missing closing quote in string literal
3 | 'hello' f'world {x}
4 | 2 + 2
1 | 'hello' 'world
| ^^^^^^ Syntax Error: missing closing quote in string literal
2 | 1 + 1
3 | 'hello' f'world {x}
|


Expand Down
14 changes: 6 additions & 8 deletions ...ts/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string_multiline.py.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,14 +216,12 @@ Module(


|
6 | (
7 | 'first'
8 | 'second
| _____^
9 | | f'third'
| |_^ Syntax Error: missing closing quote in string literal
10 | )
11 | 2 + 2
6 | (
7 | 'first'
8 | 'second
| ^^^^^^^ Syntax Error: missing closing quote in string literal
9 | f'third'
10 | )
|


Expand Down
12 changes: 5 additions & 7 deletions ...uff_python_parser/tests/snapshots/invalid_syntax@re_lexing__fstring_format_spec_1.py.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,13 +324,11 @@ Module(


|
5 | f'middle {'string':\
6 | 'format spec'}
| _____________________^
7 | |
| |_^ Syntax Error: missing closing quote in string literal
8 | f'middle {'string':\\
9 | 'format spec'}
5 | f'middle {'string':\
6 | 'format spec'}
| ^^ Syntax Error: missing closing quote in string literal
7 |
8 | f'middle {'string':\\
|


Expand Down

0 comments on commit d930e97

Please sign in to comment.