diff --git a/crates/ruff_benchmark/benches/formatter.rs b/crates/ruff_benchmark/benches/formatter.rs index 98c3a97f2c956e..d24b30fe300e36 100644 --- a/crates/ruff_benchmark/benches/formatter.rs +++ b/crates/ruff_benchmark/benches/formatter.rs @@ -5,9 +5,7 @@ use ruff_benchmark::criterion::{ }; use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError}; use ruff_python_formatter::{format_module_ast, PreviewMode, PyFormatOptions}; -use ruff_python_index::CommentRangesBuilder; -use ruff_python_parser::lexer::lex; -use ruff_python_parser::{allocate_tokens_vec, parse_tokens, Mode}; +use ruff_python_parser::{parse, Mode}; #[cfg(target_os = "windows")] #[global_allocator] @@ -52,28 +50,20 @@ fn benchmark_formatter(criterion: &mut Criterion) { BenchmarkId::from_parameter(case.name()), &case, |b, case| { - let mut tokens = allocate_tokens_vec(case.code()); - let mut comment_ranges = CommentRangesBuilder::default(); - - for result in lex(case.code(), Mode::Module) { - let (token, range) = result.expect("Input to be a valid python program."); - - comment_ranges.visit_token(&token, range); - tokens.push(Ok((token, range))); - } - - let comment_ranges = comment_ranges.finish(); - - // Parse the AST. - let module = parse_tokens(tokens, case.code(), Mode::Module) - .expect("Input to be a valid python program"); + // Parse the source. + let program = + parse(case.code(), Mode::Module).expect("Input should be a valid Python code"); b.iter(|| { let options = PyFormatOptions::from_extension(Path::new(case.name())) .with_preview(PreviewMode::Enabled); - let formatted = - format_module_ast(&module, &comment_ranges, case.code(), options) - .expect("Formatting to succeed"); + let formatted = format_module_ast( + program.syntax(), + program.comment_ranges(), + case.code(), + options, + ) + .expect("Formatting to succeed"); formatted.print().expect("Printing to succeed") }); diff --git a/crates/ruff_benchmark/benches/linter.rs b/crates/ruff_benchmark/benches/linter.rs index 60c56093f4da26..8a32f530ef3847 100644 --- a/crates/ruff_benchmark/benches/linter.rs +++ b/crates/ruff_benchmark/benches/linter.rs @@ -10,7 +10,7 @@ use ruff_linter::settings::{flags, LinterSettings}; use ruff_linter::source_kind::SourceKind; use ruff_linter::{registry::Rule, RuleSelector}; use ruff_python_ast::PySourceType; -use ruff_python_parser::{parse_program_tokens, tokenize, Mode}; +use ruff_python_parser::{parse_module, Mode}; #[cfg(target_os = "windows")] #[global_allocator] @@ -54,11 +54,9 @@ fn benchmark_linter(mut group: BenchmarkGroup, settings: &LinterSettings) { BenchmarkId::from_parameter(case.name()), &case, |b, case| { - // Tokenize the source. - let tokens = tokenize(case.code(), Mode::Module); - // Parse the source. - let ast = parse_program_tokens(tokens.clone(), case.code(), false).unwrap(); + let program = + parse_module(case.code()).expect("Input should be a valid Python code"); b.iter(|| { let path = case.path(); @@ -69,10 +67,7 @@ fn benchmark_linter(mut group: BenchmarkGroup, settings: &LinterSettings) { flags::Noqa::Enabled, &SourceKind::Python(case.code().to_string()), PySourceType::from(path.as_path()), - ParseSource::Precomputed { - tokens: &tokens, - ast: &ast, - }, + ParseSource::Precomputed(program), ); // Assert that file contains no parse errors diff --git a/crates/ruff_benchmark/benches/parser.rs b/crates/ruff_benchmark/benches/parser.rs index 0aca2772f937ad..3c24c8144178b2 100644 --- a/crates/ruff_benchmark/benches/parser.rs +++ b/crates/ruff_benchmark/benches/parser.rs @@ -4,7 +4,7 @@ use ruff_benchmark::criterion::{ use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError}; use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor}; use ruff_python_ast::Stmt; -use ruff_python_parser::parse_suite; +use ruff_python_parser::{parse_module, Mode}; #[cfg(target_os = "windows")] #[global_allocator] @@ -60,7 +60,9 @@ fn benchmark_parser(criterion: &mut Criterion) { &case, |b, case| { b.iter(|| { - let parsed = parse_suite(case.code()).unwrap(); + let parsed = parse_module(case.code()) + .expect("Input should be a valid Python code") + .into_suite(); let mut visitor = CountVisitor { count: 0 }; visitor.visit_body(&parsed); diff --git a/crates/ruff_dev/src/print_ast.rs b/crates/ruff_dev/src/print_ast.rs index cb72d0403abaa8..35206ca45ec135 100644 --- a/crates/ruff_dev/src/print_ast.rs +++ b/crates/ruff_dev/src/print_ast.rs @@ -24,7 +24,7 @@ pub(crate) fn main(args: &Args) -> Result<()> { args.file.display() ) })?; - let python_ast = parse(source_kind.source_code(), source_type.as_mode())?; + let python_ast = parse(source_kind.source_code(), source_type.as_mode())?.into_syntax(); println!("{python_ast:#?}"); Ok(()) } diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index 53a51485635f43..a590d6221752bd 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -32,8 +32,10 @@ use itertools::Itertools; use log::debug; use ruff_python_ast::{ self as ast, AnyParameterRef, Comprehension, ElifElseClause, ExceptHandler, Expr, ExprContext, - FStringElement, Keyword, MatchCase, Parameter, Parameters, Pattern, Stmt, Suite, UnaryOp, + FStringElement, Keyword, MatchCase, ModModule, Parameter, Parameters, Pattern, Stmt, Suite, + UnaryOp, }; +use ruff_python_parser::Program; use ruff_text_size::{Ranged, TextRange, TextSize}; use ruff_diagnostics::{Diagnostic, IsolationLevel}; @@ -174,6 +176,8 @@ impl ExpectedDocstringKind { } pub(crate) struct Checker<'a> { + /// The parsed [`Program`]. + program: &'a Program, /// The [`Path`] to the file under analysis. path: &'a Path, /// The [`Path`] to the package containing the current file. @@ -223,6 +227,7 @@ pub(crate) struct Checker<'a> { impl<'a> Checker<'a> { #[allow(clippy::too_many_arguments)] pub(crate) fn new( + program: &'a Program, settings: &'a LinterSettings, noqa_line_for: &'a NoqaMapping, noqa: flags::Noqa, @@ -232,12 +237,12 @@ impl<'a> Checker<'a> { locator: &'a Locator, stylist: &'a Stylist, indexer: &'a Indexer, - importer: Importer<'a>, source_type: PySourceType, cell_offsets: Option<&'a CellOffsets>, notebook_index: Option<&'a NotebookIndex>, ) -> Checker<'a> { Checker { + program, settings, noqa_line_for, noqa, @@ -248,7 +253,7 @@ impl<'a> Checker<'a> { locator, stylist, indexer, - importer, + importer: Importer::new(program.suite(), locator, stylist), semantic: SemanticModel::new(&settings.typing_modules, path, module), visit: deferred::Visit::default(), analyze: deferred::Analyze::default(), @@ -318,6 +323,11 @@ impl<'a> Checker<'a> { } } + /// The [`Program`] for the current file, which contains the tokens, AST, and more. + pub(crate) const fn program(&self) -> &'a Program { + self.program + } + /// The [`Locator`] for the current file, which enables extraction of source code from byte /// offsets. pub(crate) const fn locator(&self) -> &'a Locator<'a> { @@ -2321,7 +2331,7 @@ impl<'a> Checker<'a> { #[allow(clippy::too_many_arguments)] pub(crate) fn check_ast( - python_ast: &Suite, + program: &Program, locator: &Locator, stylist: &Stylist, indexer: &Indexer, @@ -2351,10 +2361,11 @@ pub(crate) fn check_ast( } else { ModuleSource::File(path) }, - python_ast, + python_ast: program.suite(), }; let mut checker = Checker::new( + program, settings, noqa_line_for, noqa, @@ -2364,7 +2375,6 @@ pub(crate) fn check_ast( locator, stylist, indexer, - Importer::new(python_ast, locator, stylist), source_type, cell_offsets, notebook_index, @@ -2372,8 +2382,8 @@ pub(crate) fn check_ast( checker.bind_builtins(); // Iterate over the AST. - checker.visit_module(python_ast); - checker.visit_body(python_ast); + checker.visit_module(program.suite()); + checker.visit_body(program.suite()); // Visit any deferred syntax nodes. Take care to visit in order, such that we avoid adding // new deferred nodes after visiting nodes of that kind. For example, visiting a deferred diff --git a/crates/ruff_linter/src/fix/edits.rs b/crates/ruff_linter/src/fix/edits.rs index 3d45f1ea01bb18..048af21a4a37aa 100644 --- a/crates/ruff_linter/src/fix/edits.rs +++ b/crates/ruff_linter/src/fix/edits.rs @@ -531,8 +531,9 @@ mod tests { use test_case::test_case; use ruff_diagnostics::{Diagnostic, Edit, Fix}; + use ruff_python_ast::Stmt; use ruff_python_codegen::Stylist; - use ruff_python_parser::{lexer, parse_expression, parse_suite, Mode}; + use ruff_python_parser::{lexer, parse, parse_expression, parse_module, Mode}; use ruff_source_file::Locator; use ruff_text_size::{Ranged, TextRange, TextSize}; @@ -541,17 +542,21 @@ mod tests { add_to_dunder_all, make_redundant_alias, next_stmt_break, trailing_semicolon, }; + /// Parse the given source using [`Mode::Module`] and return the first statement. + fn parse_first_stmt(source: &str) -> Result { + let suite = parse_module(source)?.into_suite(); + Ok(suite.into_iter().next().unwrap()) + } + #[test] fn find_semicolon() -> Result<()> { let contents = "x = 1"; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); + let stmt = parse_first_stmt(contents)?; let locator = Locator::new(contents); assert_eq!(trailing_semicolon(stmt.end(), &locator), None); let contents = "x = 1; y = 1"; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); + let stmt = parse_first_stmt(contents)?; let locator = Locator::new(contents); assert_eq!( trailing_semicolon(stmt.end(), &locator), @@ -559,8 +564,7 @@ mod tests { ); let contents = "x = 1 ; y = 1"; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); + let stmt = parse_first_stmt(contents)?; let locator = Locator::new(contents); assert_eq!( trailing_semicolon(stmt.end(), &locator), @@ -572,8 +576,7 @@ x = 1 \ ; y = 1 " .trim(); - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); + let stmt = parse_first_stmt(contents)?; let locator = Locator::new(contents); assert_eq!( trailing_semicolon(stmt.end(), &locator), @@ -614,10 +617,9 @@ x = 1 \ #[test] fn redundant_alias() { let contents = "import x, y as y, z as bees"; - let program = parse_suite(contents).unwrap(); - let stmt = program.first().unwrap(); + let stmt = parse_first_stmt(contents)?; assert_eq!( - make_redundant_alias(["x"].into_iter().map(Cow::from), stmt), + make_redundant_alias(["x"].into_iter().map(Cow::from), &stmt), vec![Edit::range_replacement( String::from("x as x"), TextRange::new(TextSize::new(7), TextSize::new(8)), @@ -625,7 +627,7 @@ x = 1 \ "make just one item redundant" ); assert_eq!( - make_redundant_alias(vec!["x", "y"].into_iter().map(Cow::from), stmt), + make_redundant_alias(vec!["x", "y"].into_iter().map(Cow::from), &stmt), vec![Edit::range_replacement( String::from("x as x"), TextRange::new(TextSize::new(7), TextSize::new(8)), @@ -633,7 +635,7 @@ x = 1 \ "the second item is already a redundant alias" ); assert_eq!( - make_redundant_alias(vec!["x", "z"].into_iter().map(Cow::from), stmt), + make_redundant_alias(vec!["x", "z"].into_iter().map(Cow::from), &stmt), vec![Edit::range_replacement( String::from("x as x"), TextRange::new(TextSize::new(7), TextSize::new(8)), @@ -661,13 +663,13 @@ x = 1 \ fn add_to_dunder_all_test(raw: &str, names: &[&str], expect: &str) -> Result<()> { let locator = Locator::new(raw); let edits = { - let expr = parse_expression(raw)?; + let expr = parse_expression(raw)?.expr(); let stylist = Stylist::from_tokens( &lexer::lex(raw, Mode::Expression).collect::>(), &locator, ); // SUT - add_to_dunder_all(names.iter().copied(), &expr, &stylist) + add_to_dunder_all(names.iter().copied(), expr, &stylist) }; let diag = { use crate::rules::pycodestyle::rules::MissingNewlineAtEndOfFile; diff --git a/crates/ruff_linter/src/importer/insertion.rs b/crates/ruff_linter/src/importer/insertion.rs index 274147a756e840..b4cd082ee6d488 100644 --- a/crates/ruff_linter/src/importer/insertion.rs +++ b/crates/ruff_linter/src/importer/insertion.rs @@ -321,7 +321,7 @@ mod tests { use ruff_python_ast::PySourceType; use ruff_python_codegen::Stylist; - use ruff_python_parser::{parse_suite, Mode}; + use ruff_python_parser::{parse, parse_module, Mode}; use ruff_source_file::{LineEnding, Locator}; use ruff_text_size::TextSize; @@ -330,11 +330,11 @@ mod tests { #[test] fn start_of_file() -> Result<()> { fn insert(contents: &str) -> Result { - let program = parse_suite(contents)?; + let suite = parse_module(contents)?.into_suite(); let tokens = ruff_python_parser::tokenize(contents, Mode::Module); let locator = Locator::new(contents); let stylist = Stylist::from_tokens(&tokens, &locator); - Ok(Insertion::start_of_file(&program, &locator, &stylist)) + Ok(Insertion::start_of_file(&suite, &locator, &stylist)) } let contents = ""; diff --git a/crates/ruff_linter/src/linter.rs b/crates/ruff_linter/src/linter.rs index c8bf5656f5f5fe..cd5456c07d22fc 100644 --- a/crates/ruff_linter/src/linter.rs +++ b/crates/ruff_linter/src/linter.rs @@ -10,11 +10,11 @@ use rustc_hash::FxHashMap; use ruff_diagnostics::Diagnostic; use ruff_notebook::Notebook; -use ruff_python_ast::{PySourceType, Suite}; +use ruff_python_ast::{ModModule, PySourceType, Suite}; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; use ruff_python_parser::lexer::LexResult; -use ruff_python_parser::{AsMode, ParseError, TokenKindIter, Tokens}; +use ruff_python_parser::{AsMode, ParseError, Program, TokenKindIter, Tokens}; use ruff_source_file::{Locator, SourceFileBuilder}; use ruff_text_size::Ranged; @@ -82,7 +82,7 @@ pub fn check_path( noqa: flags::Noqa, source_kind: &SourceKind, source_type: PySourceType, - tokens: TokenSource, + program: Program, ) -> LinterResult> { // Aggregate all diagnostics. let mut diagnostics = vec![]; @@ -93,7 +93,7 @@ pub fn check_path( let use_doc_lines = settings.rules.enabled(Rule::DocLineTooLong); let mut doc_lines = vec![]; if use_doc_lines { - doc_lines.extend(doc_lines_from_tokens(tokens.kinds())); + doc_lines.extend(doc_lines_from_tokens(program.kinds())); } // Run the token-based rules. @@ -103,7 +103,7 @@ pub fn check_path( .any(|rule_code| rule_code.lint_source().is_tokens()) { diagnostics.extend(check_tokens( - &tokens, + &program, path, locator, indexer, @@ -130,7 +130,7 @@ pub fn check_path( .any(|rule_code| rule_code.lint_source().is_logical_lines()) { diagnostics.extend(crate::checkers::logical_lines::check_logical_lines( - &tokens, locator, indexer, stylist, settings, + &program, locator, indexer, stylist, settings, )); } @@ -145,14 +145,13 @@ pub fn check_path( .iter_enabled() .any(|rule_code| rule_code.lint_source().is_imports()); if use_ast || use_imports || use_doc_lines { - // Parse, if the AST wasn't pre-provided provided. - match tokens.into_ast_source(source_kind, source_type) { - Ok(python_ast) => { + match program.into_result() { + Ok(program) => { let cell_offsets = source_kind.as_ipy_notebook().map(Notebook::cell_offsets); let notebook_index = source_kind.as_ipy_notebook().map(Notebook::index); if use_ast { diagnostics.extend(check_ast( - &python_ast, + &program, locator, stylist, indexer, @@ -168,7 +167,7 @@ pub fn check_path( } if use_imports { let import_diagnostics = check_imports( - &python_ast, + program.suite(), locator, indexer, &directives.isort, @@ -182,7 +181,7 @@ pub fn check_path( diagnostics.extend(import_diagnostics); } if use_doc_lines { - doc_lines.extend(doc_lines_from_ast(&python_ast, locator)); + doc_lines.extend(doc_lines_from_ast(program.suite(), locator)); } } Err(parse_error) => { @@ -350,23 +349,22 @@ pub fn add_noqa_to_path( source_type: PySourceType, settings: &LinterSettings, ) -> Result { - let contents = source_kind.source_code(); - - // Tokenize once. - let tokens = ruff_python_parser::tokenize(contents, source_type.as_mode()); + // Parse once. + let program = + ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); // Map row and column locations to byte slices (lazily). - let locator = Locator::new(contents); + let locator = Locator::new(source_kind.source_code()); // Detect the current code style (lazily). - let stylist = Stylist::from_tokens(&tokens, &locator); + let stylist = Stylist::from_tokens(&program, &locator); // Extra indices from the code. - let indexer = Indexer::from_tokens(&tokens, &locator); + let indexer = Indexer::from_tokens(&program, &locator); // Extract the `# noqa` and `# isort: skip` directives from the source. let directives = directives::extract_directives( - &tokens, + &program, directives::Flags::from_settings(settings), &locator, &indexer, @@ -387,7 +385,7 @@ pub fn add_noqa_to_path( flags::Noqa::Disabled, source_kind, source_type, - TokenSource::Tokens(tokens), + program, ); // Log any parse errors. @@ -425,23 +423,22 @@ pub fn lint_only( noqa: flags::Noqa, source_kind: &SourceKind, source_type: PySourceType, - data: ParseSource, + source: ParseSource, ) -> LinterResult> { - // Tokenize once. - let tokens = data.into_token_source(source_kind, source_type); + let program = source.into_program(source_kind, source_type); // Map row and column locations to byte slices (lazily). let locator = Locator::new(source_kind.source_code()); // Detect the current code style (lazily). - let stylist = Stylist::from_tokens(&tokens, &locator); + let stylist = Stylist::from_tokens(&program, &locator); // Extra indices from the code. - let indexer = Indexer::from_tokens(&tokens, &locator); + let indexer = Indexer::from_tokens(&program, &locator); // Extract the `# noqa` and `# isort: skip` directives from the source. let directives = directives::extract_directives( - &tokens, + &program, directives::Flags::from_settings(settings), &locator, &indexer, @@ -459,7 +456,7 @@ pub fn lint_only( noqa, source_kind, source_type, - tokens, + program, ); result.map(|diagnostics| diagnostics_to_messages(diagnostics, path, &locator, &directives)) @@ -517,21 +514,22 @@ pub fn lint_fix<'a>( // Continuously fix until the source code stabilizes. loop { - // Tokenize once. - let tokens = ruff_python_parser::tokenize(transformed.source_code(), source_type.as_mode()); + // Parse once. + let program = + ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); // Map row and column locations to byte slices (lazily). let locator = Locator::new(transformed.source_code()); // Detect the current code style (lazily). - let stylist = Stylist::from_tokens(&tokens, &locator); + let stylist = Stylist::from_tokens(&program, &locator); // Extra indices from the code. - let indexer = Indexer::from_tokens(&tokens, &locator); + let indexer = Indexer::from_tokens(&program, &locator); // Extract the `# noqa` and `# isort: skip` directives from the source. let directives = directives::extract_directives( - &tokens, + &program, directives::Flags::from_settings(settings), &locator, &indexer, @@ -549,7 +547,7 @@ pub fn lint_fix<'a>( noqa, &transformed, source_type, - TokenSource::Tokens(tokens), + program, ); if iterations == 0 { @@ -684,100 +682,27 @@ This indicates a bug in Ruff. If you could open an issue at: } #[derive(Debug, Clone)] -pub enum ParseSource<'a> { - /// Extract the tokens and AST from the given source code. +pub enum ParseSource { + /// Parse the [`Program`] from the given source code. None, - /// Use the precomputed tokens and AST. - Precomputed { - tokens: &'a [LexResult], - ast: &'a Suite, - }, -} - -impl<'a> ParseSource<'a> { - /// Convert to a [`TokenSource`], tokenizing if necessary. - fn into_token_source( - self, - source_kind: &SourceKind, - source_type: PySourceType, - ) -> TokenSource<'a> { - match self { - Self::None => TokenSource::Tokens(ruff_python_parser::tokenize( - source_kind.source_code(), - source_type.as_mode(), - )), - Self::Precomputed { tokens, ast } => TokenSource::Precomputed { tokens, ast }, - } - } -} - -#[derive(Debug, Clone)] -pub enum TokenSource<'a> { - /// Use the precomputed tokens to generate the AST. - Tokens(Tokens), - /// Use the precomputed tokens and AST. - Precomputed { - tokens: &'a [LexResult], - ast: &'a Suite, - }, -} - -impl TokenSource<'_> { - /// Returns an iterator over the [`TokenKind`] and the corresponding range. - /// - /// [`TokenKind`]: ruff_python_parser::TokenKind - pub fn kinds(&self) -> TokenKindIter { - match self { - TokenSource::Tokens(tokens) => tokens.kinds(), - TokenSource::Precomputed { tokens, .. } => TokenKindIter::new(tokens), - } - } + /// Use the precomputed [`Program`]. + Precomputed(Program), } -impl Deref for TokenSource<'_> { - type Target = [LexResult]; - - fn deref(&self) -> &Self::Target { - match self { - Self::Tokens(tokens) => tokens, - Self::Precomputed { tokens, .. } => tokens, - } - } -} - -impl<'a> TokenSource<'a> { - /// Convert to an [`AstSource`], parsing if necessary. - fn into_ast_source( +impl ParseSource { + /// Consumes the [`ParseSource`] and returns the parsed [`Program`], parsing the source code if + /// necessary. + fn into_program( self, source_kind: &SourceKind, source_type: PySourceType, - ) -> Result, ParseError> { + ) -> Program { match self { - Self::Tokens(tokens) => Ok(AstSource::Ast(ruff_python_parser::parse_program_tokens( - tokens, - source_kind.source_code(), - source_type.is_ipynb(), - )?)), - Self::Precomputed { ast, .. } => Ok(AstSource::Precomputed(ast)), - } - } -} - -#[derive(Debug, Clone)] -pub enum AstSource<'a> { - /// Extract the AST from the given source code. - Ast(Suite), - /// Use the precomputed AST. - Precomputed(&'a Suite), -} - -impl Deref for AstSource<'_> { - type Target = Suite; - - fn deref(&self) -> &Self::Target { - match self { - Self::Ast(ast) => ast, - Self::Precomputed(ast) => ast, + // SAFETY: Safe because `PySourceType` always parses to a `ModModule` + ParseSource::None => { + ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type) + } + ParseSource::Precomputed(program) => program, } } } diff --git a/crates/ruff_linter/src/rules/eradicate/detection.rs b/crates/ruff_linter/src/rules/eradicate/detection.rs index fa870287c65729..f2e02296683675 100644 --- a/crates/ruff_linter/src/rules/eradicate/detection.rs +++ b/crates/ruff_linter/src/rules/eradicate/detection.rs @@ -4,7 +4,7 @@ use itertools::Itertools; use once_cell::sync::Lazy; use regex::{Regex, RegexSet}; -use ruff_python_parser::parse_suite; +use ruff_python_parser::parse_module; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::TextSize; @@ -84,7 +84,7 @@ pub(crate) fn comment_contains_code(line: &str, task_tags: &[String]) -> bool { } // Finally, compile the source code. - parse_suite(line).is_ok() + parse_module(line).is_ok() } #[cfg(test)] diff --git a/crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs b/crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs index d8564dd5b87351..902735c2bcc67c 100644 --- a/crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs +++ b/crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs @@ -6,7 +6,7 @@ use ruff_python_ast::helpers::is_docstring_stmt; use ruff_python_ast::imports::{Alias, AnyImport, FutureImport, Import, ImportFrom}; use ruff_python_ast::{self as ast, PySourceType, Stmt, Suite}; use ruff_python_codegen::Stylist; -use ruff_python_parser::parse_suite; +use ruff_python_parser::{parse_module, Program}; use ruff_source_file::Locator; use ruff_text_size::{TextRange, TextSize}; @@ -135,7 +135,7 @@ pub(crate) fn add_required_imports( .required_imports .iter() .flat_map(|required_import| { - let Ok(body) = parse_suite(required_import) else { + let Ok(body) = parse_module(required_import).map(Program::into_suite) else { error!("Failed to parse required import: `{}`", required_import); return vec![]; }; diff --git a/crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs b/crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs index f9586b975b6d11..1fc644434c22d1 100644 --- a/crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs +++ b/crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs @@ -177,10 +177,15 @@ pub(crate) fn function_is_too_complex( mod tests { use anyhow::Result; - use ruff_python_parser::parse_suite; + use ruff_python_ast::Suite; + use ruff_python_parser::parse_module; use super::get_complexity_number; + fn parse_suite(source: &str) -> Result { + parse_module(source).map(|program| program.into_suite()) + } + #[test] fn trivial() -> Result<()> { let source = r" diff --git a/crates/ruff_linter/src/rules/pyflakes/mod.rs b/crates/ruff_linter/src/rules/pyflakes/mod.rs index 107cb96fb3769d..39dafece94a1e0 100644 --- a/crates/ruff_linter/src/rules/pyflakes/mod.rs +++ b/crates/ruff_linter/src/rules/pyflakes/mod.rs @@ -637,12 +637,13 @@ mod tests { let source_type = PySourceType::default(); let source_kind = SourceKind::Python(contents.to_string()); let settings = LinterSettings::for_rules(Linter::Pyflakes.rules()); - let tokens = ruff_python_parser::tokenize(&contents, source_type.as_mode()); + let program = + ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); let locator = Locator::new(&contents); - let stylist = Stylist::from_tokens(&tokens, &locator); - let indexer = Indexer::from_tokens(&tokens, &locator); + let stylist = Stylist::from_tokens(&program, &locator); + let indexer = Indexer::from_tokens(&program, &locator); let directives = directives::extract_directives( - &tokens, + &program, directives::Flags::from_settings(&settings), &locator, &indexer, @@ -661,7 +662,7 @@ mod tests { flags::Noqa::Enabled, &source_kind, source_type, - TokenSource::Tokens(tokens), + program, ); diagnostics.sort_by_key(Ranged::start); let actual = diagnostics diff --git a/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs b/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs index aaf4761679b4c2..c25206435bf53a 100644 --- a/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs +++ b/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs @@ -269,9 +269,9 @@ mod tests { #[test] fn extract_cmp_op_location() -> Result<()> { let contents = "x == 1"; - let expr = parse_expression(contents)?; + let expr = parse_expression(contents)?.expr(); assert_eq!( - locate_cmp_ops(&expr, contents), + locate_cmp_ops(expr, contents), vec![LocatedCmpOp::new( TextSize::from(2)..TextSize::from(4), CmpOp::Eq @@ -279,9 +279,9 @@ mod tests { ); let contents = "x != 1"; - let expr = parse_expression(contents)?; + let expr = parse_expression(contents)?.expr(); assert_eq!( - locate_cmp_ops(&expr, contents), + locate_cmp_ops(expr, contents), vec![LocatedCmpOp::new( TextSize::from(2)..TextSize::from(4), CmpOp::NotEq @@ -289,9 +289,9 @@ mod tests { ); let contents = "x is 1"; - let expr = parse_expression(contents)?; + let expr = parse_expression(contents)?.expr(); assert_eq!( - locate_cmp_ops(&expr, contents), + locate_cmp_ops(expr, contents), vec![LocatedCmpOp::new( TextSize::from(2)..TextSize::from(4), CmpOp::Is @@ -299,9 +299,9 @@ mod tests { ); let contents = "x is not 1"; - let expr = parse_expression(contents)?; + let expr = parse_expression(contents)?.expr(); assert_eq!( - locate_cmp_ops(&expr, contents), + locate_cmp_ops(expr, contents), vec![LocatedCmpOp::new( TextSize::from(2)..TextSize::from(8), CmpOp::IsNot @@ -309,9 +309,9 @@ mod tests { ); let contents = "x in 1"; - let expr = parse_expression(contents)?; + let expr = parse_expression(contents)?.expr(); assert_eq!( - locate_cmp_ops(&expr, contents), + locate_cmp_ops(expr, contents), vec![LocatedCmpOp::new( TextSize::from(2)..TextSize::from(4), CmpOp::In @@ -319,9 +319,9 @@ mod tests { ); let contents = "x not in 1"; - let expr = parse_expression(contents)?; + let expr = parse_expression(contents)?.expr(); assert_eq!( - locate_cmp_ops(&expr, contents), + locate_cmp_ops(expr, contents), vec![LocatedCmpOp::new( TextSize::from(2)..TextSize::from(8), CmpOp::NotIn @@ -329,9 +329,9 @@ mod tests { ); let contents = "x != (1 is not 2)"; - let expr = parse_expression(contents)?; + let expr = parse_expression(contents)?.expr(); assert_eq!( - locate_cmp_ops(&expr, contents), + locate_cmp_ops(expr, contents), vec![LocatedCmpOp::new( TextSize::from(2)..TextSize::from(4), CmpOp::NotEq diff --git a/crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs b/crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs index 409c85f109b2c2..482d8a9744ce35 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs @@ -254,13 +254,13 @@ pub(crate) fn too_many_branches( #[cfg(test)] mod tests { use anyhow::Result; - use ruff_python_parser::parse_suite; + use ruff_python_parser::parse_module; use super::num_branches; fn test_helper(source: &str, expected_num_branches: usize) -> Result<()> { - let branches = parse_suite(source)?; - assert_eq!(num_branches(&branches), expected_num_branches); + let branches = parse_module(source)?.suite(); + assert_eq!(num_branches(branches), expected_num_branches); Ok(()) } diff --git a/crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs b/crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs index 39f573bb11fe7f..767fd0fa33b77f 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs @@ -98,13 +98,13 @@ pub(crate) fn too_many_return_statements( #[cfg(test)] mod tests { use anyhow::Result; - use ruff_python_parser::parse_suite; + use ruff_python_parser::parse_module; use super::num_returns; fn test_helper(source: &str, expected: usize) -> Result<()> { - let stmts = parse_suite(source)?; - assert_eq!(num_returns(&stmts), expected); + let stmts = parse_module(source)?.suite(); + assert_eq!(num_returns(stmts), expected); Ok(()) } diff --git a/crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs b/crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs index d2dbf632ac67f8..ce434da1174034 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs @@ -158,10 +158,16 @@ pub(crate) fn too_many_statements( #[cfg(test)] mod tests { use anyhow::Result; - use ruff_python_parser::parse_suite; + + use ruff_python_ast::Suite; + use ruff_python_parser::parse_module; use super::num_statements; + fn parse_suite(source: &str) -> Result { + parse_module(source).map(|program| program.into_suite()) + } + #[test] fn pass() -> Result<()> { let source: &str = r" diff --git a/crates/ruff_linter/src/rules/ruff/rules/missing_fstring_syntax.rs b/crates/ruff_linter/src/rules/ruff/rules/missing_fstring_syntax.rs index 35976dbf5bf0c1..e060a63d3fbba7 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/missing_fstring_syntax.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/missing_fstring_syntax.rs @@ -114,10 +114,12 @@ fn should_be_fstring( } let fstring_expr = format!("f{}", locator.slice(literal)); + let Ok(program) = parse_expression(&fstring_expr) else { + return false; + }; // Note: Range offsets for `value` are based on `fstring_expr` - let Ok(ast::Expr::FString(ast::ExprFString { value, .. })) = parse_expression(&fstring_expr) - else { + let Some(ast::ExprFString { value, .. }) = program.expr().as_f_string_expr() else { return false; }; diff --git a/crates/ruff_linter/src/test.rs b/crates/ruff_linter/src/test.rs index b646d76bd191c3..f227c3d8120bdf 100644 --- a/crates/ruff_linter/src/test.rs +++ b/crates/ruff_linter/src/test.rs @@ -110,12 +110,13 @@ pub(crate) fn test_contents<'a>( settings: &LinterSettings, ) -> (Vec, Cow<'a, SourceKind>) { let source_type = PySourceType::from(path); - let tokens = ruff_python_parser::tokenize(source_kind.source_code(), source_type.as_mode()); + let program = + ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); let locator = Locator::new(source_kind.source_code()); - let stylist = Stylist::from_tokens(&tokens, &locator); - let indexer = Indexer::from_tokens(&tokens, &locator); + let stylist = Stylist::from_tokens(&program, &locator); + let indexer = Indexer::from_tokens(&program, &locator); let directives = directives::extract_directives( - &tokens, + &program, directives::Flags::from_settings(settings), &locator, &indexer, @@ -135,7 +136,7 @@ pub(crate) fn test_contents<'a>( flags::Noqa::Enabled, source_kind, source_type, - TokenSource::Tokens(tokens), + program, ); let source_has_errors = error.is_some(); @@ -175,13 +176,13 @@ pub(crate) fn test_contents<'a>( transformed = Cow::Owned(transformed.updated(fixed_contents, &source_map)); - let tokens = - ruff_python_parser::tokenize(transformed.source_code(), source_type.as_mode()); + let program = + ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); let locator = Locator::new(transformed.source_code()); - let stylist = Stylist::from_tokens(&tokens, &locator); - let indexer = Indexer::from_tokens(&tokens, &locator); + let stylist = Stylist::from_tokens(&program, &locator); + let indexer = Indexer::from_tokens(&program, &locator); let directives = directives::extract_directives( - &tokens, + &program, directives::Flags::from_settings(settings), &locator, &indexer, @@ -201,7 +202,7 @@ pub(crate) fn test_contents<'a>( flags::Noqa::Enabled, &transformed, source_type, - TokenSource::Tokens(tokens), + program, ); if let Some(fixed_error) = fixed_error { diff --git a/crates/ruff_python_ast_integration_tests/tests/identifier.rs b/crates/ruff_python_ast_integration_tests/tests/identifier.rs index 1e70c4fd65f50b..324390b8454c95 100644 --- a/crates/ruff_python_ast_integration_tests/tests/identifier.rs +++ b/crates/ruff_python_ast_integration_tests/tests/identifier.rs @@ -1,5 +1,5 @@ use ruff_python_ast::identifier; -use ruff_python_parser::{parse_suite, ParseError}; +use ruff_python_parser::{parse_module, ParseError}; use ruff_text_size::{TextRange, TextSize}; #[test] @@ -11,7 +11,7 @@ else: pass " .trim(); - let stmts = parse_suite(contents)?; + let stmts = parse_module(contents)?.into_suite(); let stmt = stmts.first().unwrap(); let range = identifier::else_(stmt, contents).unwrap(); assert_eq!(&contents[range], "else"); diff --git a/crates/ruff_python_ast_integration_tests/tests/parenthesize.rs b/crates/ruff_python_ast_integration_tests/tests/parenthesize.rs index 6e6b2eeaaa07e7..c877261fd2eb3a 100644 --- a/crates/ruff_python_ast_integration_tests/tests/parenthesize.rs +++ b/crates/ruff_python_ast_integration_tests/tests/parenthesize.rs @@ -6,9 +6,9 @@ use ruff_text_size::TextRange; #[test] fn test_parenthesized_name() { let source_code = r"(x) + 1"; - let expr = parse_expression(source_code).unwrap(); + let program = parse_expression(source_code).unwrap(); - let bin_op = expr.as_bin_op_expr().unwrap(); + let bin_op = program.expr().as_bin_op_expr().unwrap(); let name = bin_op.left.as_ref(); let parenthesized = parenthesized_range( @@ -23,9 +23,9 @@ fn test_parenthesized_name() { #[test] fn test_non_parenthesized_name() { let source_code = r"x + 1"; - let expr = parse_expression(source_code).unwrap(); + let program = parse_expression(source_code).unwrap(); - let bin_op = expr.as_bin_op_expr().unwrap(); + let bin_op = program.expr().as_bin_op_expr().unwrap(); let name = bin_op.left.as_ref(); let parenthesized = parenthesized_range( @@ -40,9 +40,9 @@ fn test_non_parenthesized_name() { #[test] fn test_parenthesized_argument() { let source_code = r"f((a))"; - let expr = parse_expression(source_code).unwrap(); + let program = parse_expression(source_code).unwrap(); - let call = expr.as_call_expr().unwrap(); + let call = program.expr().as_call_expr().unwrap(); let arguments = &call.arguments; let argument = arguments.args.first().unwrap(); @@ -58,9 +58,9 @@ fn test_parenthesized_argument() { #[test] fn test_non_parenthesized_argument() { let source_code = r"f(a)"; - let expr = parse_expression(source_code).unwrap(); + let program = parse_expression(source_code).unwrap(); - let call = expr.as_call_expr().unwrap(); + let call = program.expr().as_call_expr().unwrap(); let arguments = &call.arguments; let argument = arguments.args.first().unwrap(); @@ -76,9 +76,9 @@ fn test_non_parenthesized_argument() { #[test] fn test_parenthesized_tuple_member() { let source_code = r"(a, (b))"; - let expr = parse_expression(source_code).unwrap(); + let program = parse_expression(source_code).unwrap(); - let tuple = expr.as_tuple_expr().unwrap(); + let tuple = program.expr().as_tuple_expr().unwrap(); let member = tuple.elts.last().unwrap(); let parenthesized = parenthesized_range( @@ -93,9 +93,9 @@ fn test_parenthesized_tuple_member() { #[test] fn test_non_parenthesized_tuple_member() { let source_code = r"(a, b)"; - let expr = parse_expression(source_code).unwrap(); + let program = parse_expression(source_code).unwrap(); - let tuple = expr.as_tuple_expr().unwrap(); + let tuple = program.expr().as_tuple_expr().unwrap(); let member = tuple.elts.last().unwrap(); let parenthesized = parenthesized_range( @@ -110,9 +110,9 @@ fn test_non_parenthesized_tuple_member() { #[test] fn test_twice_parenthesized_name() { let source_code = r"((x)) + 1"; - let expr = parse_expression(source_code).unwrap(); + let program = parse_expression(source_code).unwrap(); - let bin_op = expr.as_bin_op_expr().unwrap(); + let bin_op = program.expr().as_bin_op_expr().unwrap(); let name = bin_op.left.as_ref(); let parenthesized = parenthesized_range( @@ -127,9 +127,9 @@ fn test_twice_parenthesized_name() { #[test] fn test_twice_parenthesized_argument() { let source_code = r"f(((a + 1)))"; - let expr = parse_expression(source_code).unwrap(); + let program = parse_expression(source_code).unwrap(); - let call = expr.as_call_expr().unwrap(); + let call = program.expr().as_call_expr().unwrap(); let arguments = &call.arguments; let argument = arguments.args.first().unwrap(); diff --git a/crates/ruff_python_ast_integration_tests/tests/preorder.rs b/crates/ruff_python_ast_integration_tests/tests/preorder.rs index 21a159b424a3a4..8c375da3e0ed25 100644 --- a/crates/ruff_python_ast_integration_tests/tests/preorder.rs +++ b/crates/ruff_python_ast_integration_tests/tests/preorder.rs @@ -4,8 +4,7 @@ use insta::assert_snapshot; use ruff_python_ast::visitor::preorder::{PreorderVisitor, TraversalSignal}; use ruff_python_ast::{AnyNodeRef, BoolOp, CmpOp, Operator, Singleton, UnaryOp}; -use ruff_python_parser::lexer::lex; -use ruff_python_parser::{parse_tokens, Mode}; +use ruff_python_parser::{parse, Mode}; #[test] fn function_arguments() { @@ -148,11 +147,10 @@ fn f_strings() { } fn trace_preorder_visitation(source: &str) -> String { - let tokens = lex(source, Mode::Module); - let parsed = parse_tokens(tokens.collect(), source, Mode::Module).unwrap(); + let parsed = parse(source, Mode::Module).unwrap(); let mut visitor = RecordVisitor::default(); - visitor.visit_mod(&parsed); + visitor.visit_mod(parsed.syntax()); visitor.output } diff --git a/crates/ruff_python_ast_integration_tests/tests/stmt_if.rs b/crates/ruff_python_ast_integration_tests/tests/stmt_if.rs index cacf964996f5b5..fd416e6845b4a3 100644 --- a/crates/ruff_python_ast_integration_tests/tests/stmt_if.rs +++ b/crates/ruff_python_ast_integration_tests/tests/stmt_if.rs @@ -1,5 +1,5 @@ use ruff_python_ast::stmt_if::elif_else_range; -use ruff_python_parser::{parse_suite, ParseError}; +use ruff_python_parser::{parse_module, ParseError}; use ruff_text_size::TextSize; #[test] @@ -9,12 +9,14 @@ fn extract_elif_else_range() -> Result<(), ParseError> { elif b: ... "; - let mut stmts = parse_suite(contents)?; - let stmt = stmts - .pop() - .and_then(ruff_python_ast::Stmt::if_stmt) - .unwrap(); - let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap(); + let program = parse_module(contents)?; + let if_stmt = program + .suite() + .first() + .expect("module should contain at least one statement") + .as_if_stmt() + .expect("first statement should be an `if` statement"); + let range = elif_else_range(&if_stmt.elif_else_clauses[0], contents).unwrap(); assert_eq!(range.start(), TextSize::from(14)); assert_eq!(range.end(), TextSize::from(18)); @@ -23,12 +25,14 @@ elif b: else: ... "; - let mut stmts = parse_suite(contents)?; - let stmt = stmts - .pop() - .and_then(ruff_python_ast::Stmt::if_stmt) - .unwrap(); - let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap(); + let program = parse_module(contents)?; + let if_stmt = program + .suite() + .first() + .expect("module should contain at least one statement") + .as_if_stmt() + .expect("first statement should be an `if` statement"); + let range = elif_else_range(&if_stmt.elif_else_clauses[0], contents).unwrap(); assert_eq!(range.start(), TextSize::from(14)); assert_eq!(range.end(), TextSize::from(18)); diff --git a/crates/ruff_python_ast_integration_tests/tests/visitor.rs b/crates/ruff_python_ast_integration_tests/tests/visitor.rs index 1c1bf0d0f7fb38..128d0c3f121848 100644 --- a/crates/ruff_python_ast_integration_tests/tests/visitor.rs +++ b/crates/ruff_python_ast_integration_tests/tests/visitor.rs @@ -13,8 +13,7 @@ use ruff_python_ast::{ Expr, FString, FStringElement, Keyword, MatchCase, Operator, Parameter, Parameters, Pattern, Stmt, StringLiteral, TypeParam, UnaryOp, WithItem, }; -use ruff_python_parser::lexer::lex; -use ruff_python_parser::{parse_tokens, Mode}; +use ruff_python_parser::{parse, Mode}; #[test] fn function_arguments() { @@ -157,11 +156,10 @@ fn f_strings() { } fn trace_visitation(source: &str) -> String { - let tokens = lex(source, Mode::Module); - let parsed = parse_tokens(tokens.collect(), source, Mode::Module).unwrap(); + let parsed = parse(source, Mode::Module).unwrap(); let mut visitor = RecordVisitor::default(); - walk_module(&mut visitor, &parsed); + walk_module(&mut visitor, parsed.syntax()); visitor.output } diff --git a/crates/ruff_python_codegen/src/generator.rs b/crates/ruff_python_codegen/src/generator.rs index 1c95db1f9c3c07..c7d8ba74d7f06e 100644 --- a/crates/ruff_python_codegen/src/generator.rs +++ b/crates/ruff_python_codegen/src/generator.rs @@ -1416,7 +1416,7 @@ impl<'a> Generator<'a> { #[cfg(test)] mod tests { use ruff_python_ast::{str::Quote, Mod, ModModule}; - use ruff_python_parser::{self, parse_suite, Mode}; + use ruff_python_parser::{self, parse_module, Mode}; use ruff_source_file::LineEnding; use crate::stylist::Indentation; @@ -1427,9 +1427,9 @@ mod tests { let indentation = Indentation::default(); let quote = Quote::default(); let line_ending = LineEnding::default(); - let stmt = parse_suite(contents).unwrap(); + let module = parse_module(contents).unwrap(); let mut generator = Generator::new(&indentation, quote, line_ending); - generator.unparse_suite(&stmt); + generator.unparse_suite(module.suite()); generator.generate() } @@ -1439,9 +1439,9 @@ mod tests { line_ending: LineEnding, contents: &str, ) -> String { - let stmt = parse_suite(contents).unwrap(); + let module = parse_module(contents).unwrap(); let mut generator = Generator::new(indentation, quote, line_ending); - generator.unparse_suite(&stmt); + generator.unparse_suite(module.suite()); generator.generate() } @@ -1449,8 +1449,8 @@ mod tests { let indentation = Indentation::default(); let quote = Quote::default(); let line_ending = LineEnding::default(); - let ast = ruff_python_parser::parse(contents, Mode::Ipython).unwrap(); - let Mod::Module(ModModule { body, .. }) = ast else { + let program = ruff_python_parser::parse(contents, Mode::Ipython).unwrap(); + let Mod::Module(ModModule { body, .. }) = program.into_syntax() else { panic!("Source code didn't return ModModule") }; let [stmt] = body.as_slice() else { diff --git a/crates/ruff_python_codegen/src/lib.rs b/crates/ruff_python_codegen/src/lib.rs index baa71ea1278fb4..1e94ecfa0da44f 100644 --- a/crates/ruff_python_codegen/src/lib.rs +++ b/crates/ruff_python_codegen/src/lib.rs @@ -2,17 +2,17 @@ mod generator; mod stylist; pub use generator::Generator; -use ruff_python_parser::{lexer, parse_suite, Mode, ParseError}; +use ruff_python_parser::{lexer, parse_module, Mode, ParseError}; use ruff_source_file::Locator; pub use stylist::Stylist; /// Run round-trip source code generation on a given Python code. pub fn round_trip(code: &str) -> Result { let locator = Locator::new(code); - let python_ast = parse_suite(code)?; + let stmts = parse_module(code)?.suite(); let tokens: Vec<_> = lexer::lex(code, Mode::Module).collect(); let stylist = Stylist::from_tokens(&tokens, &locator); let mut generator: Generator = (&stylist).into(); - generator.unparse_suite(&python_ast); + generator.unparse_suite(&stmts); Ok(generator.generate()) } diff --git a/crates/ruff_python_formatter/src/cli.rs b/crates/ruff_python_formatter/src/cli.rs index 881ca8ffae6663..012c3024ddf8e2 100644 --- a/crates/ruff_python_formatter/src/cli.rs +++ b/crates/ruff_python_formatter/src/cli.rs @@ -7,8 +7,7 @@ use clap::{command, Parser, ValueEnum}; use ruff_formatter::SourceCode; use ruff_python_ast::PySourceType; -use ruff_python_index::tokens_and_ranges; -use ruff_python_parser::{parse_tokens, AsMode}; +use ruff_python_parser::{parse, AsMode}; use ruff_text_size::Ranged; use crate::comments::collect_comments; @@ -46,12 +45,9 @@ pub struct Cli { pub fn format_and_debug_print(source: &str, cli: &Cli, source_path: &Path) -> Result { let source_type = PySourceType::from(source_path); - let (tokens, comment_ranges) = tokens_and_ranges(source, source_type) - .map_err(|err| format_err!("Source contains syntax errors {err:?}"))?; // Parse the AST. - let module = - parse_tokens(tokens, source, source_type.as_mode()).context("Syntax error in input")?; + let program = parse(source, source_type.as_mode()).context("Syntax error in input")?; let options = PyFormatOptions::from_extension(source_path) .with_preview(if cli.preview { @@ -66,14 +62,15 @@ pub fn format_and_debug_print(source: &str, cli: &Cli, source_path: &Path) -> Re }); let source_code = SourceCode::new(source); - let formatted = format_module_ast(&module, &comment_ranges, source, options) + let formatted = format_module_ast(program.syntax(), program.comment_ranges(), source, options) .context("Failed to format node")?; if cli.print_ir { println!("{}", formatted.document().display(source_code)); } if cli.print_comments { // Print preceding, following and enclosing nodes - let decorated_comments = collect_comments(&module, source_code, &comment_ranges); + let decorated_comments = + collect_comments(program.syntax(), source_code, program.comment_ranges()); if !decorated_comments.is_empty() { println!("# Comment decoration: Range, Preceding, Following, Enclosing, Comment"); } diff --git a/crates/ruff_python_formatter/src/comments/mod.rs b/crates/ruff_python_formatter/src/comments/mod.rs index 9717252a9b4358..ef775a11179672 100644 --- a/crates/ruff_python_formatter/src/comments/mod.rs +++ b/crates/ruff_python_formatter/src/comments/mod.rs @@ -481,15 +481,14 @@ mod tests { use ruff_formatter::SourceCode; use ruff_python_ast::{Mod, PySourceType}; - use ruff_python_index::tokens_and_ranges; - use ruff_python_parser::{parse_tokens, AsMode}; + use ruff_python_parser::{parse, AsMode}; use ruff_python_trivia::CommentRanges; use crate::comments::Comments; struct CommentsTestCase<'a> { - module: Mod, - comment_ranges: CommentRanges, + module: &'a Mod, + comment_ranges: &'a CommentRanges, source_code: SourceCode<'a>, } @@ -497,15 +496,13 @@ mod tests { fn from_code(source: &'a str) -> Self { let source_code = SourceCode::new(source); let source_type = PySourceType::Python; - let (tokens, comment_ranges) = - tokens_and_ranges(source, source_type).expect("Expect source to be valid Python"); - let parsed = parse_tokens(tokens, source, source_type.as_mode()) - .expect("Expect source to be valid Python"); + let program = + parse(source, source_type.as_mode()).expect("Expect source to be valid Python"); CommentsTestCase { source_code, - module: parsed, - comment_ranges, + module: program.syntax(), + comment_ranges: program.comment_ranges(), } } diff --git a/crates/ruff_python_formatter/src/expression/expr_name.rs b/crates/ruff_python_formatter/src/expression/expr_name.rs index 276ded6dd9a50d..5a8b6b2665089e 100644 --- a/crates/ruff_python_formatter/src/expression/expr_name.rs +++ b/crates/ruff_python_formatter/src/expression/expr_name.rs @@ -31,15 +31,15 @@ impl NeedsParentheses for ExprName { #[cfg(test)] mod tests { - use ruff_python_parser::parse_program; + use ruff_python_parser::parse_module; use ruff_text_size::{Ranged, TextRange, TextSize}; #[test] fn name_range_with_comments() { - let source = parse_program("a # comment").unwrap(); + let module = parse_module("a # comment").unwrap(); - let expression_statement = source - .body + let expression_statement = module + .suite() .first() .expect("Expected non-empty body") .as_expr_stmt() diff --git a/crates/ruff_python_formatter/src/expression/parentheses.rs b/crates/ruff_python_formatter/src/expression/parentheses.rs index 766bb7071a3236..0003459a3ca814 100644 --- a/crates/ruff_python_formatter/src/expression/parentheses.rs +++ b/crates/ruff_python_formatter/src/expression/parentheses.rs @@ -444,17 +444,16 @@ impl Format> for FormatEmptyParenthesized<'_> { mod tests { use ruff_python_ast::ExpressionRef; use ruff_python_parser::parse_expression; - use ruff_python_trivia::CommentRanges; use crate::expression::parentheses::is_expression_parenthesized; #[test] fn test_has_parentheses() { let expression = r#"(b().c("")).d()"#; - let expr = parse_expression(expression).unwrap(); + let program = parse_expression(expression).unwrap(); assert!(!is_expression_parenthesized( - ExpressionRef::from(&expr), - &CommentRanges::default(), + ExpressionRef::from(program.expr()), + program.comment_ranges(), expression )); } diff --git a/crates/ruff_python_formatter/src/lib.rs b/crates/ruff_python_formatter/src/lib.rs index 858714abf8dcbe..c35c9caa0d29b2 100644 --- a/crates/ruff_python_formatter/src/lib.rs +++ b/crates/ruff_python_formatter/src/lib.rs @@ -6,8 +6,7 @@ use ruff_formatter::prelude::*; use ruff_formatter::{format, write, FormatError, Formatted, PrintError, Printed, SourceCode}; use ruff_python_ast::AstNode; use ruff_python_ast::Mod; -use ruff_python_index::tokens_and_ranges; -use ruff_python_parser::{parse_tokens, AsMode, ParseError, ParseErrorType}; +use ruff_python_parser::{parse, AsMode, ParseError, ParseErrorType}; use ruff_python_trivia::CommentRanges; use ruff_source_file::Locator; @@ -114,13 +113,8 @@ pub fn format_module_source( options: PyFormatOptions, ) -> Result { let source_type = options.source_type(); - let (tokens, comment_ranges) = - tokens_and_ranges(source, source_type).map_err(|err| ParseError { - location: err.location(), - error: ParseErrorType::Lexical(err.into_error()), - })?; - let module = parse_tokens(tokens, source, source_type.as_mode())?; - let formatted = format_module_ast(&module, &comment_ranges, source, options)?; + let program = parse(source, source_type.as_mode())?; + let formatted = format_module_ast(program.syntax(), program.comment_ranges(), source, options)?; Ok(formatted.print()?) } @@ -161,8 +155,7 @@ mod tests { use insta::assert_snapshot; use ruff_python_ast::PySourceType; - use ruff_python_index::tokens_and_ranges; - use ruff_python_parser::{parse_tokens, AsMode}; + use ruff_python_parser::{parse, AsMode}; use ruff_text_size::{TextRange, TextSize}; use crate::{format_module_ast, format_module_source, format_range, PyFormatOptions}; @@ -203,13 +196,13 @@ def main() -> None: "#; let source_type = PySourceType::Python; - let (tokens, comment_ranges) = tokens_and_ranges(source, source_type).unwrap(); // Parse the AST. let source_path = "code_inline.py"; - let module = parse_tokens(tokens, source, source_type.as_mode()).unwrap(); + let program = parse(source, source_type.as_mode()).unwrap(); let options = PyFormatOptions::from_extension(Path::new(source_path)); - let formatted = format_module_ast(&module, &comment_ranges, source, options).unwrap(); + let formatted = + format_module_ast(program.syntax(), program.comment_ranges(), source, options).unwrap(); // Uncomment the `dbg` to print the IR. // Use `dbg_write!(f, []) instead of `write!(f, [])` in your formatting code to print some IR diff --git a/crates/ruff_python_formatter/src/range.rs b/crates/ruff_python_formatter/src/range.rs index 58ea00117c8ba8..5c63b179b04ffc 100644 --- a/crates/ruff_python_formatter/src/range.rs +++ b/crates/ruff_python_formatter/src/range.rs @@ -7,7 +7,7 @@ use ruff_formatter::{ use ruff_python_ast::visitor::preorder::{walk_body, PreorderVisitor, TraversalSignal}; use ruff_python_ast::{AnyNode, AnyNodeRef, Stmt, StmtMatch, StmtTry}; use ruff_python_index::tokens_and_ranges; -use ruff_python_parser::{parse_tokens, AsMode, ParseError, ParseErrorType}; +use ruff_python_parser::{parse, parse_tokens, AsMode, ParseError, ParseErrorType}; use ruff_python_trivia::{indentation_at_offset, BackwardsTokenizer, SimpleToken, SimpleTokenKind}; use ruff_source_file::Locator; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; @@ -71,18 +71,11 @@ pub fn format_range( return Ok(PrintedRange::new(formatted.into_code(), range)); } - let (tokens, comment_ranges) = - tokens_and_ranges(source, options.source_type()).map_err(|err| ParseError { - location: err.location(), - error: ParseErrorType::Lexical(err.into_error()), - })?; - assert_valid_char_boundaries(range, source); - let module = parse_tokens(tokens, source, options.source_type().as_mode())?; - let root = AnyNode::from(module); + let program = parse(source, options.source_type().as_mode())?; let source_code = SourceCode::new(source); - let comments = Comments::from_ast(root.as_ref(), source_code, &comment_ranges); + let comments = Comments::from_ast(program.syntax(), source_code, program.comment_ranges()); let mut context = PyFormatContext::new( options.with_source_map_generation(SourceMapGeneration::Enabled), @@ -90,13 +83,14 @@ pub fn format_range( comments, ); - let (enclosing_node, base_indent) = match find_enclosing_node(range, root.as_ref(), &context) { - EnclosingNode::Node { node, indent_level } => (node, indent_level), - EnclosingNode::Suppressed => { - // The entire range falls into a suppressed range. There's nothing to format. - return Ok(PrintedRange::empty()); - } - }; + let (enclosing_node, base_indent) = + match find_enclosing_node(range, AnyNodeRef::from(program.syntax()), &context) { + EnclosingNode::Node { node, indent_level } => (node, indent_level), + EnclosingNode::Suppressed => { + // The entire range falls into a suppressed range. There's nothing to format. + return Ok(PrintedRange::empty()); + } + }; let narrowed_range = narrow_range(range, enclosing_node, &context); assert_valid_char_boundaries(narrowed_range, source); diff --git a/crates/ruff_python_formatter/src/statement/suite.rs b/crates/ruff_python_formatter/src/statement/suite.rs index 844fcc0bec5e6a..6cf752ea6707cd 100644 --- a/crates/ruff_python_formatter/src/statement/suite.rs +++ b/crates/ruff_python_formatter/src/statement/suite.rs @@ -829,7 +829,7 @@ impl Format> for SuiteChildStatement<'_> { #[cfg(test)] mod tests { use ruff_formatter::format; - use ruff_python_parser::parse_suite; + use ruff_python_parser::parse_module; use ruff_python_trivia::CommentRanges; use crate::comments::Comments; @@ -859,17 +859,16 @@ def trailing_func(): pass "; - let statements = parse_suite(source).unwrap(); + let module = parse_module(source).unwrap(); - let comment_ranges = CommentRanges::default(); let context = PyFormatContext::new( PyFormatOptions::default(), source, - Comments::from_ranges(&comment_ranges), + Comments::from_ranges(module.comment_ranges()), ); let test_formatter = - format_with(|f: &mut PyFormatter| statements.format().with_options(level).fmt(f)); + format_with(|f: &mut PyFormatter| module.suite().format().with_options(level).fmt(f)); let formatted = format!(context, [test_formatter]).unwrap(); let printed = formatted.print().unwrap(); diff --git a/crates/ruff_python_formatter/src/string/docstring.rs b/crates/ruff_python_formatter/src/string/docstring.rs index 6aefad2a12f08a..9ae80f7f0645fe 100644 --- a/crates/ruff_python_formatter/src/string/docstring.rs +++ b/crates/ruff_python_formatter/src/string/docstring.rs @@ -1552,16 +1552,15 @@ fn docstring_format_source( use ruff_python_parser::AsMode; let source_type = options.source_type(); - let (tokens, comment_ranges) = - ruff_python_index::tokens_and_ranges(source, source_type).map_err(ParseError::from)?; - let module = ruff_python_parser::parse_tokens(tokens, source, source_type.as_mode())?; + let program = ruff_python_parser::parse(source, source_type.as_mode())?; let source_code = ruff_formatter::SourceCode::new(source); - let comments = crate::Comments::from_ast(&module, source_code, &comment_ranges); + let comments = + crate::Comments::from_ast(program.syntax(), source_code, program.comment_ranges()); let locator = Locator::new(source); let ctx = PyFormatContext::new(options, locator.contents(), comments) .in_docstring(docstring_quote_style); - let formatted = crate::format!(ctx, [module.format()])?; + let formatted = crate::format!(ctx, [program.syntax().format()])?; formatted .context() .comments() diff --git a/crates/ruff_python_formatter/tests/fixtures.rs b/crates/ruff_python_formatter/tests/fixtures.rs index a72e505e7a5385..0c4da466eeaf19 100644 --- a/crates/ruff_python_formatter/tests/fixtures.rs +++ b/crates/ruff_python_formatter/tests/fixtures.rs @@ -391,13 +391,15 @@ fn ensure_unchanged_ast( // Parse the unformatted code. let mut unformatted_ast = parse(unformatted_code, source_type.as_mode()) - .expect("Unformatted code to be valid syntax"); + .expect("Unformatted code to be valid syntax") + .into_syntax(); Normalizer.visit_module(&mut unformatted_ast); let unformatted_ast = ComparableMod::from(&unformatted_ast); // Parse the formatted code. - let mut formatted_ast = - parse(formatted_code, source_type.as_mode()).expect("Formatted code to be valid syntax"); + let mut formatted_ast = parse(formatted_code, source_type.as_mode()) + .expect("Formatted code to be valid syntax") + .into_syntax(); Normalizer.visit_module(&mut formatted_ast); let formatted_ast = ComparableMod::from(&formatted_ast); diff --git a/crates/ruff_python_index/src/comment_ranges.rs b/crates/ruff_python_index/src/comment_ranges.rs index e9ef4c04620bf1..602446a934c47e 100644 --- a/crates/ruff_python_index/src/comment_ranges.rs +++ b/crates/ruff_python_index/src/comment_ranges.rs @@ -1,8 +1,6 @@ use std::fmt::Debug; -use ruff_python_ast::PySourceType; -use ruff_python_parser::lexer::{lex, LexResult, LexicalError}; -use ruff_python_parser::{allocate_tokens_vec, AsMode, Tok}; +use ruff_python_parser::Tok; use ruff_python_trivia::CommentRanges; use ruff_text_size::TextRange; @@ -22,23 +20,3 @@ impl CommentRangesBuilder { CommentRanges::new(self.ranges) } } - -/// Helper method to lex and extract comment ranges -pub fn tokens_and_ranges( - source: &str, - source_type: PySourceType, -) -> Result<(Vec, CommentRanges), LexicalError> { - let mut tokens = allocate_tokens_vec(source); - let mut comment_ranges = CommentRangesBuilder::default(); - - for result in lex(source, source_type.as_mode()) { - if let Ok((token, range)) = &result { - comment_ranges.visit_token(token, *range); - } - - tokens.push(result); - } - - let comment_ranges = comment_ranges.finish(); - Ok((tokens, comment_ranges)) -} diff --git a/crates/ruff_python_index/src/lib.rs b/crates/ruff_python_index/src/lib.rs index 2a4660f0125cf7..7f1117de97a9db 100644 --- a/crates/ruff_python_index/src/lib.rs +++ b/crates/ruff_python_index/src/lib.rs @@ -3,5 +3,5 @@ mod fstring_ranges; mod indexer; mod multiline_ranges; -pub use comment_ranges::{tokens_and_ranges, CommentRangesBuilder}; +pub use comment_ranges::CommentRangesBuilder; pub use indexer::Indexer; diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 0c1455923a089a..280fceb45cc425 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -33,6 +33,9 @@ mod cursor; mod fstring; mod indentation; +#[deprecated] +pub fn lex(_source: &str, _mode: Mode) {} + /// A lexer for Python source code. #[derive(Debug)] pub struct Lexer<'src> { diff --git a/crates/ruff_python_parser/src/lib.rs b/crates/ruff_python_parser/src/lib.rs index cf232e8efca530..85e4b545b074dc 100644 --- a/crates/ruff_python_parser/src/lib.rs +++ b/crates/ruff_python_parser/src/lib.rs @@ -87,6 +87,9 @@ mod token_set; mod token_source; pub mod typing; +#[deprecated] +pub fn tokenize(_source: &str, _mode: Mode) {} + /// Parse a full Python module usually consisting of multiple lines. /// /// This is a convenience function that can be used to parse a full Python program without having to diff --git a/crates/ruff_python_semantic/src/analyze/type_inference.rs b/crates/ruff_python_semantic/src/analyze/type_inference.rs index e2ecd8690e3aa1..33ce82a5aba005 100644 --- a/crates/ruff_python_semantic/src/analyze/type_inference.rs +++ b/crates/ruff_python_semantic/src/analyze/type_inference.rs @@ -428,12 +428,12 @@ impl NumberLike { #[cfg(test)] mod tests { - use ruff_python_ast::Expr; - use ruff_python_parser::parse_expression; + use ruff_python_ast::{Expr, ModExpression}; + use ruff_python_parser::{parse_expression, Program}; use crate::analyze::type_inference::{NumberLike, PythonType, ResolvedPythonType}; - fn parse(expression: &str) -> Expr { + fn parse(expression: &str) -> Program { parse_expression(expression).unwrap() } @@ -441,95 +441,95 @@ mod tests { fn type_inference() { // Atoms. assert_eq!( - ResolvedPythonType::from(&parse("1")), + ResolvedPythonType::from(parse("1").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Integer)) ); assert_eq!( - ResolvedPythonType::from(&parse("'Hello, world'")), + ResolvedPythonType::from(parse("'Hello, world'").expr()), ResolvedPythonType::Atom(PythonType::String) ); assert_eq!( - ResolvedPythonType::from(&parse("b'Hello, world'")), + ResolvedPythonType::from(parse("b'Hello, world'").expr()), ResolvedPythonType::Atom(PythonType::Bytes) ); assert_eq!( - ResolvedPythonType::from(&parse("'Hello' % 'world'")), + ResolvedPythonType::from(parse("'Hello' % 'world'").expr()), ResolvedPythonType::Atom(PythonType::String) ); // Boolean operators. assert_eq!( - ResolvedPythonType::from(&parse("1 and 2")), + ResolvedPythonType::from(parse("1 and 2").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Integer)) ); assert_eq!( - ResolvedPythonType::from(&parse("1 and True")), + ResolvedPythonType::from(parse("1 and True").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Integer)) ); // Binary operators. assert_eq!( - ResolvedPythonType::from(&parse("1.0 * 2")), + ResolvedPythonType::from(parse("1.0 * 2").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Float)) ); assert_eq!( - ResolvedPythonType::from(&parse("2 * 1.0")), + ResolvedPythonType::from(parse("2 * 1.0").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Float)) ); assert_eq!( - ResolvedPythonType::from(&parse("1.0 * 2j")), + ResolvedPythonType::from(parse("1.0 * 2j").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Complex)) ); assert_eq!( - ResolvedPythonType::from(&parse("1 / True")), + ResolvedPythonType::from(parse("1 / True").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Float)) ); assert_eq!( - ResolvedPythonType::from(&parse("1 / 2")), + ResolvedPythonType::from(parse("1 / 2").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Float)) ); assert_eq!( - ResolvedPythonType::from(&parse("{1, 2} - {2}")), + ResolvedPythonType::from(parse("{1, 2} - {2}").expr()), ResolvedPythonType::Atom(PythonType::Set) ); // Unary operators. assert_eq!( - ResolvedPythonType::from(&parse("-1")), + ResolvedPythonType::from(parse("-1").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Integer)) ); assert_eq!( - ResolvedPythonType::from(&parse("-1.0")), + ResolvedPythonType::from(parse("-1.0").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Float)) ); assert_eq!( - ResolvedPythonType::from(&parse("-1j")), + ResolvedPythonType::from(parse("-1j").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Complex)) ); assert_eq!( - ResolvedPythonType::from(&parse("-True")), + ResolvedPythonType::from(parse("-True").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Integer)) ); assert_eq!( - ResolvedPythonType::from(&parse("not 'Hello'")), + ResolvedPythonType::from(parse("not 'Hello'").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Bool)) ); assert_eq!( - ResolvedPythonType::from(&parse("not x.y.z")), + ResolvedPythonType::from(parse("not x.y.z").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Bool)) ); // Conditional expressions. assert_eq!( - ResolvedPythonType::from(&parse("1 if True else 2")), + ResolvedPythonType::from(parse("1 if True else 2").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Integer)) ); assert_eq!( - ResolvedPythonType::from(&parse("1 if True else 2.0")), + ResolvedPythonType::from(parse("1 if True else 2.0").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Float)) ); assert_eq!( - ResolvedPythonType::from(&parse("1 if True else False")), + ResolvedPythonType::from(parse("1 if True else False").expr()), ResolvedPythonType::Atom(PythonType::Number(NumberLike::Integer)) ); } diff --git a/crates/ruff_python_trivia_integration_tests/tests/whitespace.rs b/crates/ruff_python_trivia_integration_tests/tests/whitespace.rs index 709a3a3d189b62..d73e2052b3ef45 100644 --- a/crates/ruff_python_trivia_integration_tests/tests/whitespace.rs +++ b/crates/ruff_python_trivia_integration_tests/tests/whitespace.rs @@ -1,4 +1,4 @@ -use ruff_python_parser::{parse_suite, ParseError}; +use ruff_python_parser::{parse_module, ParseError}; use ruff_python_trivia::has_trailing_content; use ruff_source_file::Locator; use ruff_text_size::Ranged; @@ -6,26 +6,26 @@ use ruff_text_size::Ranged; #[test] fn trailing_content() -> Result<(), ParseError> { let contents = "x = 1"; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); + let suite = parse_module(contents)?.into_suite(); + let stmt = suite.first().unwrap(); let locator = Locator::new(contents); assert!(!has_trailing_content(stmt.end(), &locator)); let contents = "x = 1; y = 2"; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); + let suite = parse_module(contents)?.into_suite(); + let stmt = suite.first().unwrap(); let locator = Locator::new(contents); assert!(has_trailing_content(stmt.end(), &locator)); let contents = "x = 1 "; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); + let suite = parse_module(contents)?.into_suite(); + let stmt = suite.first().unwrap(); let locator = Locator::new(contents); assert!(!has_trailing_content(stmt.end(), &locator)); let contents = "x = 1 # Comment"; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); + let suite = parse_module(contents)?.into_suite(); + let stmt = suite.first().unwrap(); let locator = Locator::new(contents); assert!(!has_trailing_content(stmt.end(), &locator)); @@ -34,8 +34,8 @@ x = 1 y = 2 " .trim(); - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); + let suite = parse_module(contents)?.into_suite(); + let stmt = suite.first().unwrap(); let locator = Locator::new(contents); assert!(!has_trailing_content(stmt.end(), &locator)); diff --git a/crates/ruff_server/src/lint.rs b/crates/ruff_server/src/lint.rs index 4815a9d5ef1a9e..8c157ed75e5a9b 100644 --- a/crates/ruff_server/src/lint.rs +++ b/crates/ruff_server/src/lint.rs @@ -4,7 +4,7 @@ use ruff_diagnostics::{Applicability, Diagnostic, DiagnosticKind, Edit, Fix}; use ruff_linter::{ directives::{extract_directives, Flags}, generate_noqa_edits, - linter::{check_path, LinterResult, TokenSource}, + linter::{check_path, LinterResult}, packaging::detect_package_root, registry::AsRule, settings::{flags, LinterSettings}, @@ -75,8 +75,9 @@ pub(crate) fn check( let source_type = query.source_type(); - // Tokenize once. - let tokens = ruff_python_parser::tokenize(source_kind.source_code(), source_type.as_mode()); + // Parse once. + let program = + ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); let index = LineIndex::from_source_text(source_kind.source_code()); @@ -84,13 +85,13 @@ pub(crate) fn check( let locator = Locator::with_index(source_kind.source_code(), index.clone()); // Detect the current code style (lazily). - let stylist = Stylist::from_tokens(&tokens, &locator); + let stylist = Stylist::from_tokens(&program, &locator); // Extra indices from the code. - let indexer = Indexer::from_tokens(&tokens, &locator); + let indexer = Indexer::from_tokens(&program, &locator); // Extract the `# noqa` and `# isort: skip` directives from the source. - let directives = extract_directives(&tokens, Flags::all(), &locator, &indexer); + let directives = extract_directives(&program, Flags::all(), &locator, &indexer); // Generate checks. let LinterResult { data, .. } = check_path( @@ -104,7 +105,7 @@ pub(crate) fn check( flags::Noqa::Enabled, &source_kind, source_type, - TokenSource::Tokens(tokens), + program, ); let noqa_edits = generate_noqa_edits( diff --git a/crates/ruff_wasm/src/lib.rs b/crates/ruff_wasm/src/lib.rs index f5977ece161bca..88ff6fc845a4b8 100644 --- a/crates/ruff_wasm/src/lib.rs +++ b/crates/ruff_wasm/src/lib.rs @@ -16,8 +16,8 @@ use ruff_linter::source_kind::SourceKind; use ruff_python_ast::{Mod, PySourceType}; use ruff_python_codegen::Stylist; use ruff_python_formatter::{format_module_ast, pretty_comments, PyFormatContext, QuoteStyle}; -use ruff_python_index::{CommentRangesBuilder, Indexer}; -use ruff_python_parser::{parse_tokens, tokenize_all, AsMode, Mode, Program}; +use ruff_python_index::Indexer; +use ruff_python_parser::{parse, AsMode, Mode, Program}; use ruff_python_trivia::CommentRanges; use ruff_source_file::{Locator, SourceLocation}; use ruff_text_size::Ranged; @@ -160,21 +160,26 @@ impl Workspace { // TODO(dhruvmanila): Support Jupyter Notebooks let source_kind = SourceKind::Python(contents.to_string()); - // Tokenize once. - let tokens = ruff_python_parser::tokenize(contents, source_type.as_mode()); + // Parse once. + let program = + ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); // Map row and column locations to byte slices (lazily). let locator = Locator::new(contents); // Detect the current code style (lazily). - let stylist = Stylist::from_tokens(&tokens, &locator); + let stylist = Stylist::from_tokens(&program, &locator); // Extra indices from the code. - let indexer = Indexer::from_tokens(&tokens, &locator); + let indexer = Indexer::from_tokens(&program, &locator); // Extract the `# noqa` and `# isort: skip` directives from the source. - let directives = - directives::extract_directives(&tokens, directives::Flags::empty(), &locator, &indexer); + let directives = directives::extract_directives( + &program, + directives::Flags::empty(), + &locator, + &indexer, + ); // Generate checks. let LinterResult { @@ -190,7 +195,7 @@ impl Workspace { flags::Noqa::Enabled, &source_kind, source_type, - TokenSource::Tokens(tokens), + program, ); let source_code = locator.to_source_code(); @@ -242,7 +247,11 @@ impl Workspace { pub fn comments(&self, contents: &str) -> Result { let parsed = ParsedModule::from_source(contents)?; - let comments = pretty_comments(&parsed.module, &parsed.comment_ranges, contents); + let comments = pretty_comments( + parsed.program.syntax(), + parsed.program.comment_ranges(), + contents, + ); Ok(comments) } @@ -266,25 +275,14 @@ pub(crate) fn into_error(err: E) -> Error { struct ParsedModule<'a> { source_code: &'a str, - module: Mod, - comment_ranges: CommentRanges, + program: Program, } impl<'a> ParsedModule<'a> { fn from_source(source_code: &'a str) -> Result { - let tokens: Vec<_> = tokenize_all(source_code, Mode::Module); - let mut comment_ranges = CommentRangesBuilder::default(); - - for (token, range) in tokens.iter().flatten() { - comment_ranges.visit_token(token, *range); - } - let comment_ranges = comment_ranges.finish(); - let module = parse_tokens(tokens, source_code, Mode::Module).map_err(into_error)?; - Ok(Self { source_code, - module, - comment_ranges, + program: parse(source_code, Mode::Module).map_err(into_error)?, }) } @@ -296,8 +294,8 @@ impl<'a> ParsedModule<'a> { .with_source_map_generation(SourceMapGeneration::Enabled); format_module_ast( - &self.module, - &self.comment_ranges, + self.program.syntax(), + self.program.comment_ranges(), self.source_code, options, )