Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Proper f string formatting 5 #1

Draft
wants to merge 23 commits into
base: joined-string-part
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
08998dd
ast for JoinedStringPart
davidszotten Aug 8, 2023
dc2b9fd
FStringPart for parser/string.rs
davidszotten Aug 5, 2023
bc5f2e6
FStringPart for formatter (no change yet; formatter only uses FString…
davidszotten Aug 5, 2023
0117b06
format_spec -> FStringPart instead of Expr
davidszotten Aug 5, 2023
35c815b
FStringPart for generator
davidszotten Aug 5, 2023
2d64b27
wip fixing ruff rules
davidszotten Aug 5, 2023
37af3b6
fix with feature=unreachable-code
davidszotten Aug 6, 2023
293cc5c
FormattedValue is a node
davidszotten Aug 17, 2023
6558886
StringTodoName is a node
davidszotten Aug 17, 2023
6f5b384
FStringPart: String(StringTodoName) -> Literal(PartialString)
davidszotten Aug 20, 2023
54afa75
generate AsFormat and friends
davidszotten Aug 20, 2023
65030af
remove accidental pub
davidszotten Aug 21, 2023
5b0b7d0
maybe this is enough for proper f-string formatting
davidszotten Aug 24, 2023
7688773
inline partialstring here to avoid allocating
davidszotten Aug 25, 2023
d828b31
more test cases
davidszotten Aug 25, 2023
1337db1
more tests
davidszotten Aug 25, 2023
eb429e7
remove old todo
davidszotten Aug 25, 2023
a4d9307
double-curly avoidance not needed if there is debug_text
davidszotten Aug 25, 2023
7b882ef
allow changing quotes inside triple-quotes
davidszotten Aug 25, 2023
78f25a7
snapshots
davidszotten Aug 25, 2023
8ff9059
ok to change quotes in formatted values if no escaping is needed
davidszotten Aug 26, 2023
193690f
actually need to consider _all surrounding quotes
davidszotten Aug 26, 2023
cbb5c64
try not to break lines inside f-strings
davidszotten Aug 27, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions crates/ruff/src/checkers/ast/analyze/expression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -931,15 +931,15 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
pylint::rules::await_outside_async(checker, expr);
}
}
Expr::FString(ast::ExprFString { values, .. }) => {
Expr::FString(ast::ExprFString { parts, .. }) => {
if checker.enabled(Rule::FStringMissingPlaceholders) {
pyflakes::rules::f_string_missing_placeholders(expr, values, checker);
pyflakes::rules::f_string_missing_placeholders(expr, parts, checker);
}
if checker.enabled(Rule::HardcodedSQLExpression) {
flake8_bandit::rules::hardcoded_sql_expression(checker, expr);
}
if checker.enabled(Rule::ExplicitFStringTypeConversion) {
ruff::rules::explicit_f_string_type_conversion(checker, expr, values);
ruff::rules::explicit_f_string_type_conversion(checker, expr, parts);
}
}
Expr::BinOp(ast::ExprBinOp {
Expand Down
11 changes: 0 additions & 11 deletions crates/ruff/src/checkers/ast/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1283,17 +1283,6 @@ where
self.semantic.flags = flags_snapshot;
}

fn visit_format_spec(&mut self, format_spec: &'b Expr) {
match format_spec {
Expr::FString(ast::ExprFString { values, .. }) => {
for value in values {
self.visit_expr(value);
}
}
_ => unreachable!("Unexpected expression for format_spec"),
}
}

fn visit_parameters(&mut self, parameters: &'b Parameters) {
// Step 1: Binding.
// Bind, but intentionally avoid walking default expressions, as we handle them
Expand Down
2 changes: 1 addition & 1 deletion crates/ruff/src/rules/flake8_comprehensions/fixes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1004,7 +1004,7 @@ pub(crate) fn fix_unnecessary_map(
// If the expression is embedded in an f-string, surround it with spaces to avoid
// syntax errors.
if matches!(object_type, ObjectType::Set | ObjectType::Dict) {
if parent.is_some_and(Expr::is_formatted_value_expr) {
if parent.is_some_and(Expr::is_f_string_expr) {
content = format!(" {content} ");
}
}
Expand Down
13 changes: 11 additions & 2 deletions crates/ruff/src/rules/flake8_pytest_style/rules/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,22 @@ pub(super) fn is_empty_or_null_string(expr: &Expr) -> bool {
..
}) => string.is_empty(),
Expr::Constant(constant) if constant.value.is_none() => true,
Expr::FString(ast::ExprFString { values, .. }) => {
values.iter().all(is_empty_or_null_string)
Expr::FString(ast::ExprFString { parts, .. }) => {
parts.iter().all(is_empty_or_null_string_part)
}
_ => false,
}
}

fn is_empty_or_null_string_part(part: &ast::FStringPart) -> bool {
match part {
ast::FStringPart::Literal(ast::PartialString { value, .. }) => value.is_empty(),
ast::FStringPart::FormattedValue(ast::FormattedValue { expression, .. }) => {
is_empty_or_null_string(expression)
}
}
}

pub(super) fn split_names(names: &str) -> Vec<&str> {
// Match the following pytest code:
// [x.strip() for x in argnames.split(",") if x.strip()]
Expand Down
33 changes: 16 additions & 17 deletions crates/ruff/src/rules/flynt/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,22 @@ use ruff_python_ast::{self as ast, Arguments, Constant, ConversionFlag, Expr};
use ruff_text_size::TextRange;

/// Wrap an expression in a `FormattedValue` with no special formatting.
fn to_formatted_value_expr(inner: &Expr) -> Expr {
let node = ast::ExprFormattedValue {
value: Box::new(inner.clone()),
fn to_formatted_value_expr(inner: &Expr) -> ast::FStringPart {
ast::FStringPart::FormattedValue(ast::FormattedValue {
expression: Box::new(inner.clone()),
debug_text: None,
conversion: ConversionFlag::None,
format_spec: None,
format_spec: vec![],
range: TextRange::default(),
};
node.into()
})
}

/// Convert a string to a constant string expression.
pub(super) fn to_constant_string(s: &str) -> Expr {
let node = ast::ExprConstant {
value: s.to_owned().into(),
kind: None,
pub(super) fn to_constant_string(s: &str) -> ast::FStringPart {
ast::FStringPart::Literal(ast::PartialString {
value: s.to_owned(),
range: TextRange::default(),
};
node.into()
})
}

/// Figure out if `expr` represents a "simple" call
Expand Down Expand Up @@ -52,15 +49,17 @@ fn is_simple_callee(func: &Expr) -> bool {
}

/// Convert an expression to a f-string element (if it looks like a good idea).
pub(super) fn to_f_string_element(expr: &Expr) -> Option<Expr> {
pub(super) fn to_fstring_part(expr: &Expr) -> Option<ast::FStringPart> {
match expr {
// These are directly handled by `unparse_f_string_element`:
Expr::Constant(ast::ExprConstant {
value: Constant::Str(_),
value: Constant::Str(value),
range,
..
})
| Expr::FString(_)
| Expr::FormattedValue(_) => Some(expr.clone()),
}) => Some(ast::FStringPart::Literal(ast::PartialString {
value: value.to_string(),
range: *range,
})),
// These should be pretty safe to wrap in a formatted value.
Expr::Constant(ast::ExprConstant {
value:
Expand Down
8 changes: 4 additions & 4 deletions crates/ruff/src/rules/flynt/rules/static_join_to_fstring.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ fn build_fstring(joiner: &str, joinees: &[Expr]) -> Option<Expr> {
return Some(node.into());
}

let mut fstring_elems = Vec::with_capacity(joinees.len() * 2);
let mut fstring_parts = Vec::with_capacity(joinees.len() * 2);
let mut first = true;

for expr in joinees {
Expand All @@ -101,13 +101,13 @@ fn build_fstring(joiner: &str, joinees: &[Expr]) -> Option<Expr> {
return None;
}
if !std::mem::take(&mut first) {
fstring_elems.push(helpers::to_constant_string(joiner));
fstring_parts.push(helpers::to_constant_string(joiner));
}
fstring_elems.push(helpers::to_f_string_element(expr)?);
fstring_parts.push(helpers::to_fstring_part(expr)?);
}

let node = ast::ExprFString {
values: fstring_elems,
parts: fstring_parts,
implicit_concatenated: false,
range: TextRange::default(),
};
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use ruff_python_ast::{Expr, PySourceType, Ranged};
use ruff_python_ast::{Expr, FStringPart, PySourceType, Ranged};
use ruff_python_parser::{lexer, AsMode, StringKind, Tok};
use ruff_text_size::{TextRange, TextSize};

Expand Down Expand Up @@ -80,10 +80,14 @@ fn find_useless_f_strings<'a>(
}

/// F541
pub(crate) fn f_string_missing_placeholders(expr: &Expr, values: &[Expr], checker: &mut Checker) {
pub(crate) fn f_string_missing_placeholders(
expr: &Expr,
values: &[FStringPart],
checker: &mut Checker,
) {
if !values
.iter()
.any(|value| matches!(value, Expr::FormattedValue(_)))
.any(|value| matches!(value, FStringPart::FormattedValue(_)))
{
for (prefix_range, tok_range) in
find_useless_f_strings(expr, checker.locator(), checker.source_type)
Expand Down
26 changes: 11 additions & 15 deletions crates/ruff/src/rules/pylint/rules/assert_on_string_literal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,25 +71,21 @@ pub(crate) fn assert_on_string_literal(checker: &mut Checker, test: &Expr) {
}
_ => {}
},
Expr::FString(ast::ExprFString { values, .. }) => {
Expr::FString(ast::ExprFString { parts, .. }) => {
checker.diagnostics.push(Diagnostic::new(
AssertOnStringLiteral {
kind: if values.iter().all(|value| match value {
Expr::Constant(ast::ExprConstant { value, .. }) => match value {
Constant::Str(value, ..) => value.is_empty(),
Constant::Bytes(value) => value.is_empty(),
_ => false,
},
_ => false,
kind: if parts.iter().all(|part| match part {
ast::FStringPart::Literal(ast::PartialString { value, .. }) => {
value.is_empty()
}
ast::FStringPart::FormattedValue(_) => false,
}) {
Kind::Empty
} else if values.iter().any(|value| match value {
Expr::Constant(ast::ExprConstant { value, .. }) => match value {
Constant::Str(value, ..) => !value.is_empty(),
Constant::Bytes(value) => !value.is_empty(),
_ => false,
},
_ => false,
} else if parts.iter().any(|part| match part {
ast::FStringPart::Literal(ast::PartialString { value, .. }) => {
!value.is_empty()
}
ast::FStringPart::FormattedValue(_) => false,
}) {
Kind::NonEmpty
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,6 @@ fn is_allowed_value(expr: &Expr) -> bool {
| Expr::GeneratorExp(_)
| Expr::Compare(_)
| Expr::Call(_)
| Expr::FormattedValue(_)
| Expr::FString(_)
| Expr::Constant(_)
| Expr::Attribute(_)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,21 +55,23 @@ impl AlwaysAutofixableViolation for ExplicitFStringTypeConversion {
pub(crate) fn explicit_f_string_type_conversion(
checker: &mut Checker,
expr: &Expr,
values: &[Expr],
values: &[ast::FStringPart],
) {
for (index, formatted_value) in values
.iter()
.filter_map(|expr| {
if let Expr::FormattedValue(expr) = &expr {
if let ast::FStringPart::FormattedValue(expr) = &expr {
Some(expr)
} else {
None
}
})
.enumerate()
{
let ast::ExprFormattedValue {
value, conversion, ..
let ast::FormattedValue {
expression,
conversion,
..
} = formatted_value;

// Skip if there's already a conversion flag.
Expand All @@ -86,7 +88,7 @@ pub(crate) fn explicit_f_string_type_conversion(
range: _,
},
..
}) = value.as_ref()
}) = expression.as_ref()
else {
continue;
};
Expand Down Expand Up @@ -121,7 +123,7 @@ pub(crate) fn explicit_f_string_type_conversion(
continue;
}

let mut diagnostic = Diagnostic::new(ExplicitFStringTypeConversion, value.range());
let mut diagnostic = Diagnostic::new(ExplicitFStringTypeConversion, expression.range());
if checker.patch(diagnostic.kind.rule()) {
diagnostic.try_set_fix(|| {
convert_call_to_conversion_flag(expr, index, checker.locator(), checker.stylist())
Expand Down
1 change: 0 additions & 1 deletion crates/ruff/src/rules/ruff/rules/unreachable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -625,7 +625,6 @@ impl<'stmt> BasicBlocksBuilder<'stmt> {
| Expr::Set(_)
| Expr::Compare(_)
| Expr::Call(_)
| Expr::FormattedValue(_)
| Expr::FString(_)
| Expr::Constant(_)
| Expr::Attribute(_)
Expand Down
10 changes: 6 additions & 4 deletions crates/ruff/src/rules/tryceratops/rules/raise_vanilla_args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,12 @@ pub(crate) fn raise_vanilla_args(checker: &mut Checker, expr: &Expr) {
/// some whitespace).
fn contains_message(expr: &Expr) -> bool {
match expr {
Expr::FString(ast::ExprFString { values, .. }) => {
for value in values {
if contains_message(value) {
return true;
Expr::FString(ast::ExprFString { parts, .. }) => {
for part in parts {
if let ast::FStringPart::Literal(ast::PartialString { value, .. }) = part {
if value.chars().any(char::is_whitespace) {
return true;
}
}
}
}
Expand Down
50 changes: 34 additions & 16 deletions crates/ruff_python_ast/src/comparable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -517,6 +517,36 @@ impl<'a> From<&'a ast::ExceptHandler> for ComparableExceptHandler<'a> {
}
}

#[derive(Debug, PartialEq, Eq, Hash)]
pub enum ComparableFStringPart<'a> {
Literal(&'a str),
FormattedValue(FormattedValue<'a>),
}

#[derive(Debug, PartialEq, Eq, Hash)]
pub struct FormattedValue<'a> {
expression: ComparableExpr<'a>,
debug_text: Option<&'a ast::DebugText>,
conversion: ast::ConversionFlag,
format_spec: Vec<ComparableFStringPart<'a>>,
}

impl<'a> From<&'a ast::FStringPart> for ComparableFStringPart<'a> {
fn from(fstring_part: &'a ast::FStringPart) -> Self {
match fstring_part {
ast::FStringPart::Literal(ast::PartialString { value, .. }) => Self::Literal(value),
ast::FStringPart::FormattedValue(formatted_value) => {
Self::FormattedValue(FormattedValue {
expression: (&formatted_value.expression).into(),
debug_text: formatted_value.debug_text.as_ref(),
conversion: formatted_value.conversion,
format_spec: formatted_value.format_spec.iter().map(Into::into).collect(),
})
}
}
}
}

#[derive(Debug, PartialEq, Eq, Hash)]
pub struct ComparableElifElseClause<'a> {
test: Option<ComparableExpr<'a>>,
Expand Down Expand Up @@ -644,12 +674,12 @@ pub struct ExprFormattedValue<'a> {
value: Box<ComparableExpr<'a>>,
debug_text: Option<&'a ast::DebugText>,
conversion: ast::ConversionFlag,
format_spec: Option<Box<ComparableExpr<'a>>>,
format_spec: Vec<ComparableFStringPart<'a>>,
}

#[derive(Debug, PartialEq, Eq, Hash)]
pub struct ExprFString<'a> {
values: Vec<ComparableExpr<'a>>,
parts: Vec<ComparableFStringPart<'a>>,
}

#[derive(Debug, PartialEq, Eq, Hash)]
Expand Down Expand Up @@ -885,24 +915,12 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> {
func: func.into(),
arguments: arguments.into(),
}),
ast::Expr::FormattedValue(ast::ExprFormattedValue {
value,
conversion,
debug_text,
format_spec,
range: _,
}) => Self::FormattedValue(ExprFormattedValue {
value: value.into(),
conversion: *conversion,
debug_text: debug_text.as_ref(),
format_spec: format_spec.as_ref().map(Into::into),
}),
ast::Expr::FString(ast::ExprFString {
values,
parts,
implicit_concatenated: _,
range: _,
}) => Self::FString(ExprFString {
values: values.iter().map(Into::into).collect(),
parts: parts.iter().map(Into::into).collect(),
}),
ast::Expr::Constant(ast::ExprConstant {
value,
Expand Down
Loading