From 78c1598b55055ded6f4fefa5efe96f6e253532aa Mon Sep 17 00:00:00 2001 From: Dhruv Manilawala Date: Mon, 12 Feb 2024 23:00:02 +0530 Subject: [PATCH] Use non-parenthesized range for `DebugText` (#9953) ## Summary This PR fixes the `DebugText` implementation to use the expression range instead of the parenthesized range. Taking the following code snippet as an example: ```python x = 1 print(f"{ ( x ) = }") ``` The output of running it would be: ``` ( x ) = 1 ``` Notice that the whitespace between the parentheses and the expression is preserved as is. Currently, we don't preserve this information in the AST which defeats the purpose of `DebugText` as the main purpose of the struct is to preserve whitespaces _around_ the expression. This is also problematic when generating the code from the AST node as then the generator has no information about the parentheses the whitespaces between them and the expression which would lead to the removal of the parentheses in the generated code. I noticed this while working on the f-string formatting where the debug text would be used to preserve the text surrounding the expression in the presence of debug expression. The parentheses were being dropped then which made me realize that the problem is instead in the parser. ## Test Plan 1. Add a test case for the parser 2. Add a test case for the generator --- crates/ruff_python_codegen/src/generator.rs | 1 + crates/ruff_python_parser/src/parser.rs | 1 + crates/ruff_python_parser/src/python.lalrpop | 4 +- crates/ruff_python_parser/src/python.rs | 6 +-- ...ython_parser__parser__tests__fstrings.snap | 41 +++++++++++++++++++ 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/crates/ruff_python_codegen/src/generator.rs b/crates/ruff_python_codegen/src/generator.rs index c3d7a60ffb3e6..934a9f39847bb 100644 --- a/crates/ruff_python_codegen/src/generator.rs +++ b/crates/ruff_python_codegen/src/generator.rs @@ -1705,6 +1705,7 @@ class Foo: assert_round_trip!(r#"f"{ chr(65) = !s}""#); assert_round_trip!(r#"f"{ chr(65) = !r}""#); assert_round_trip!(r#"f"{ chr(65) = :#x}""#); + assert_round_trip!(r#"f"{ ( chr(65) ) = }""#); assert_round_trip!(r#"f"{a=!r:0.05f}""#); } diff --git a/crates/ruff_python_parser/src/parser.rs b/crates/ruff_python_parser/src/parser.rs index 46fef053bdb16..bc530a0f7d0db 100644 --- a/crates/ruff_python_parser/src/parser.rs +++ b/crates/ruff_python_parser/src/parser.rs @@ -1476,6 +1476,7 @@ f"""{ y z }""" +f"{ ( foo ) = }" "# .trim(), ) diff --git a/crates/ruff_python_parser/src/python.lalrpop b/crates/ruff_python_parser/src/python.lalrpop index f61ae2c2b4eff..34d049a0176c8 100644 --- a/crates/ruff_python_parser/src/python.lalrpop +++ b/crates/ruff_python_parser/src/python.lalrpop @@ -1656,8 +1656,8 @@ FStringReplacementField: ast::FStringElement = { ) }; ast::DebugText { - leading: source_code[TextRange::new(start_offset, value.start())].to_string(), - trailing: source_code[TextRange::new(value.end(), end_offset)].to_string(), + leading: source_code[TextRange::new(start_offset, value.expr.start())].to_string(), + trailing: source_code[TextRange::new(value.expr.end(), end_offset)].to_string(), } }); Ok( diff --git a/crates/ruff_python_parser/src/python.rs b/crates/ruff_python_parser/src/python.rs index 95de336aa7614..646fcfdeb6891 100644 --- a/crates/ruff_python_parser/src/python.rs +++ b/crates/ruff_python_parser/src/python.rs @@ -1,5 +1,5 @@ // auto-generated: "lalrpop 0.20.0" -// sha3: d38cc0f2252a58db42d3bd63a102b537865992b3cf51d402cdb4828f48989c9d +// sha3: 8c85e4bbac54760ed8be03b56a428d76e14d18e6dbde62b424d0b2b5e8e65dbe use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use ruff_python_ast::{self as ast, Int, IpyEscapeKind}; use crate::{ @@ -36457,8 +36457,8 @@ fn __action221< ) }; ast::DebugText { - leading: source_code[TextRange::new(start_offset, value.start())].to_string(), - trailing: source_code[TextRange::new(value.end(), end_offset)].to_string(), + leading: source_code[TextRange::new(start_offset, value.expr.start())].to_string(), + trailing: source_code[TextRange::new(value.expr.end(), end_offset)].to_string(), } }); Ok( diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__parser__tests__fstrings.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__parser__tests__fstrings.snap index 58c33b7302c9d..a46c03a3d9bc2 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__parser__tests__fstrings.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__parser__tests__fstrings.snap @@ -942,4 +942,45 @@ expression: parse_ast ), }, ), + Expr( + StmtExpr { + range: 374..392, + value: FString( + ExprFString { + range: 374..392, + value: FStringValue { + inner: Single( + FString( + FString { + range: 374..392, + elements: [ + Expression( + FStringExpressionElement { + range: 376..391, + expression: Name( + ExprName { + range: 381..384, + id: "foo", + ctx: Load, + }, + ), + debug_text: Some( + DebugText { + leading: " ( ", + trailing: " ) = ", + }, + ), + conversion: None, + format_spec: None, + }, + ), + ], + }, + ), + ), + }, + }, + ), + }, + ), ]