Use CommentRanges from the parsed output

crates/ruff_linter/src/checkers/filesystem.rs

@@ -1,7 +1,7 @@
 use std::path::Path;
 
 use ruff_diagnostics::Diagnostic;
-use ruff_python_index::Indexer;
+use ruff_python_trivia::CommentRanges;
 use ruff_source_file::Locator;
 
 use crate::registry::Rule;
@@ -13,7 +13,7 @@ pub(crate) fn check_file_path(
     path: &Path,
     package: Option<&Path>,
     locator: &Locator,
-    indexer: &Indexer,
+    comment_ranges: &CommentRanges,
     settings: &LinterSettings,
 ) -> Vec<Diagnostic> {
     let mut diagnostics: Vec<Diagnostic> = vec![];
@@ -24,7 +24,7 @@ pub(crate) fn check_file_path(
             path,
             package,
             locator,
-            indexer,
+            comment_ranges,
             &settings.project_root,
             &settings.src,
         ) {

crates/ruff_linter/src/checkers/imports.rs

@@ -51,7 +51,7 @@ pub(crate) fn check_imports(
                     settings,
                     package,
                     source_type,
-                    program.tokens(),
+                    program,
                 ) {
                     diagnostics.push(diagnostic);
                 }

crates/ruff_linter/src/checkers/physical_lines.rs

@@ -3,6 +3,7 @@
 use ruff_diagnostics::Diagnostic;
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
+use ruff_python_trivia::CommentRanges;
 use ruff_source_file::{Locator, UniversalNewlines};
 use ruff_text_size::TextSize;
 
@@ -19,6 +20,7 @@ pub(crate) fn check_physical_lines(
     locator: &Locator,
     stylist: &Stylist,
     indexer: &Indexer,
+    comment_ranges: &CommentRanges,
     doc_lines: &[TextSize],
     settings: &LinterSettings,
 ) -> Vec<Diagnostic> {
@@ -42,7 +44,7 @@ pub(crate) fn check_physical_lines(
             .is_some()
         {
             if enforce_doc_line_too_long {
-                if let Some(diagnostic) = doc_line_too_long(&line, indexer, settings) {
+                if let Some(diagnostic) = doc_line_too_long(&line, comment_ranges, settings) {
                     diagnostics.push(diagnostic);
                 }
             }
@@ -55,7 +57,7 @@ pub(crate) fn check_physical_lines(
         }
 
         if enforce_line_too_long {
-            if let Some(diagnostic) = line_too_long(&line, indexer, settings) {
+            if let Some(diagnostic) = line_too_long(&line, comment_ranges, settings) {
                 diagnostics.push(diagnostic);
             }
         }

crates/ruff_linter/src/checkers/tokens.rs

@@ -3,17 +3,16 @@
 use std::path::Path;
 
 use ruff_notebook::CellOffsets;
-use ruff_python_ast::PySourceType;
+use ruff_python_ast::{ModModule, PySourceType};
 use ruff_python_codegen::Stylist;
 
 use ruff_diagnostics::Diagnostic;
 use ruff_python_index::Indexer;
-use ruff_python_parser::Tokens;
+use ruff_python_parser::{Program, Tokens};
 use ruff_source_file::Locator;
 use ruff_text_size::Ranged;
 
 use crate::directives::TodoComment;
-use crate::linter::TokenSource;
 use crate::registry::{AsRule, Rule};
 use crate::rules::pycodestyle::rules::BlankLinesChecker;
 use crate::rules::{
@@ -24,7 +23,7 @@ use crate::settings::LinterSettings;
 
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn check_tokens(
-    tokens: &Tokens,
+    program: &Program<ModModule>,
     path: &Path,
     locator: &Locator,
     indexer: &Indexer,
@@ -44,22 +43,26 @@ pub(crate) fn check_tokens(
         Rule::BlankLinesBeforeNestedDefinition,
     ]) {
         BlankLinesChecker::new(locator, stylist, settings, source_type, cell_offsets)
-            .check_lines(tokens, &mut diagnostics);
+            .check_lines(program.tokens(), &mut diagnostics);
     }
 
     if settings.rules.enabled(Rule::BlanketTypeIgnore) {
-        pygrep_hooks::rules::blanket_type_ignore(&mut diagnostics, indexer, locator);
+        pygrep_hooks::rules::blanket_type_ignore(
+            &mut diagnostics,
+            program.comment_ranges(),
+            locator,
+        );
     }
 
     if settings.rules.enabled(Rule::EmptyComment) {
-        pylint::rules::empty_comments(&mut diagnostics, indexer, locator);
+        pylint::rules::empty_comments(&mut diagnostics, program.comment_ranges(), locator);
     }
 
     if settings
         .rules
         .enabled(Rule::AmbiguousUnicodeCharacterComment)
     {
-        for range in indexer.comment_ranges() {
+        for range in program.comment_ranges() {
             ruff::rules::ambiguous_unicode_character_comment(
                 &mut diagnostics,
                 locator,
@@ -70,11 +73,21 @@ pub(crate) fn check_tokens(
     }
 
     if settings.rules.enabled(Rule::CommentedOutCode) {
-        eradicate::rules::commented_out_code(&mut diagnostics, locator, indexer, settings);
+        eradicate::rules::commented_out_code(
+            &mut diagnostics,
+            locator,
+            program.comment_ranges(),
+            settings,
+        );
     }
 
     if settings.rules.enabled(Rule::UTF8EncodingDeclaration) {
-        pyupgrade::rules::unnecessary_coding_comment(&mut diagnostics, locator, indexer);
+        pyupgrade::rules::unnecessary_coding_comment(
+            &mut diagnostics,
+            locator,
+            indexer,
+            program.comment_ranges(),
+        );
     }
 
     if settings.rules.enabled(Rule::TabIndentation) {
@@ -88,7 +101,7 @@ pub(crate) fn check_tokens(
         Rule::InvalidCharacterNul,
         Rule::InvalidCharacterZeroWidthSpace,
     ]) {
-        for token in tokens.up_to_first_unknown() {
+        for token in program.tokens().up_to_first_unknown() {
             pylint::rules::invalid_string_characters(
                 &mut diagnostics,
                 token.kind(),
@@ -105,7 +118,7 @@ pub(crate) fn check_tokens(
     ]) {
         pycodestyle::rules::compound_statements(
             &mut diagnostics,
-            tokens,
+            program.tokens(),
             locator,
             indexer,
             source_type,
@@ -119,7 +132,7 @@ pub(crate) fn check_tokens(
     ]) {
         flake8_implicit_str_concat::rules::implicit(
             &mut diagnostics,
-            tokens,
+            program.tokens(),
             settings,
             locator,
             indexer,
@@ -131,15 +144,19 @@ pub(crate) fn check_tokens(
         Rule::TrailingCommaOnBareTuple,
         Rule::ProhibitedTrailingComma,
     ]) {
-        flake8_commas::rules::trailing_commas(&mut diagnostics, tokens, locator, indexer);
+        flake8_commas::rules::trailing_commas(&mut diagnostics, program.tokens(), locator, indexer);
     }
 
     if settings.rules.enabled(Rule::ExtraneousParentheses) {
-        pyupgrade::rules::extraneous_parentheses(&mut diagnostics, tokens, locator);
+        pyupgrade::rules::extraneous_parentheses(&mut diagnostics, program.tokens(), locator);
     }
 
     if source_type.is_stub() && settings.rules.enabled(Rule::TypeCommentInStub) {
-        flake8_pyi::rules::type_comment_in_stub(&mut diagnostics, locator, indexer);
+        flake8_pyi::rules::type_comment_in_stub(
+            &mut diagnostics,
+            locator,
+            program.comment_ranges(),
+        );
     }
 
     if settings.rules.any_enabled(&[
@@ -149,7 +166,12 @@ pub(crate) fn check_tokens(
         Rule::ShebangNotFirstLine,
         Rule::ShebangMissingPython,
     ]) {
-        flake8_executable::rules::from_tokens(&mut diagnostics, path, locator, indexer);
+        flake8_executable::rules::from_tokens(
+            &mut diagnostics,
+            path,
+            locator,
+            program.comment_ranges(),
+        );
     }
 
     if settings.rules.any_enabled(&[
@@ -165,7 +187,7 @@ pub(crate) fn check_tokens(
         Rule::LineContainsTodo,
         Rule::LineContainsHack,
     ]) {
-        let todo_comments: Vec<TodoComment> = indexer
+        let todo_comments: Vec<TodoComment> = program
             .comment_ranges()
             .iter()
             .enumerate()
@@ -174,12 +196,17 @@ pub(crate) fn check_tokens(
                 TodoComment::from_comment(comment, *comment_range, i)
             })
             .collect();
-        flake8_todos::rules::todos(&mut diagnostics, &todo_comments, locator, indexer);
+        flake8_todos::rules::todos(
+            &mut diagnostics,
+            &todo_comments,
+            locator,
+            program.comment_ranges(),
+        );
         flake8_fixme::rules::todos(&mut diagnostics, &todo_comments);
     }
 
     if settings.rules.enabled(Rule::TooManyNewlinesAtEndOfFile) {
-        pycodestyle::rules::too_many_newlines_at_end_of_file(&mut diagnostics, tokens);
+        pycodestyle::rules::too_many_newlines_at_end_of_file(&mut diagnostics, program.tokens());
     }
 
     diagnostics.retain(|diagnostic| settings.rules.enabled(diagnostic.kind.rule()));

crates/ruff_linter/src/directives.rs

@@ -4,9 +4,10 @@ use std::iter::Peekable;
 use std::str::FromStr;
 
 use bitflags::bitflags;
-use ruff_python_ast::StringFlags;
+use ruff_python_ast::{ModModule, StringFlags};
 use ruff_python_parser::lexer::LexResult;
-use ruff_python_parser::Tok;
+use ruff_python_parser::{Program, Tok};
+use ruff_python_trivia::CommentRanges;
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 
 use ruff_python_index::Indexer;
@@ -68,7 +69,7 @@ pub struct Directives {
 }
 
 pub fn extract_directives(
-    lxr: &[LexResult],
+    program: &Program<ModModule>,
     flags: Flags,
     locator: &Locator,
     indexer: &Indexer,
@@ -80,7 +81,7 @@ pub fn extract_directives(
             NoqaMapping::default()
         },
         isort: if flags.intersects(Flags::ISORT) {
-            extract_isort_directives(locator, indexer)
+            extract_isort_directives(locator, program.comment_ranges())
         } else {
             IsortDirectives::default()
         },
@@ -213,12 +214,12 @@ fn extract_noqa_line_for(lxr: &[LexResult], locator: &Locator, indexer: &Indexer
 }
 
 /// Extract a set of ranges over which to disable isort.
-fn extract_isort_directives(locator: &Locator, indexer: &Indexer) -> IsortDirectives {
+fn extract_isort_directives(locator: &Locator, comment_ranges: &CommentRanges) -> IsortDirectives {
     let mut exclusions: Vec<TextRange> = Vec::default();
     let mut splits: Vec<TextSize> = Vec::default();
     let mut off: Option<TextSize> = None;
 
-    for range in indexer.comment_ranges() {
+    for range in comment_ranges {
         let comment_text = locator.slice(range);
 
         // `isort` allows for `# isort: skip` and `# isort: skip_file` to include or