Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

red-knot: Use parse_unchecked to get all parse errors #11725

Merged
merged 3 commits into from
Jun 4, 2024
Merged

red-knot: Use parse_unchecked to get all parse errors #11725

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4af0673
red-knot: Use `parse_unchecked` to get all parse errors
MichaReiser Jun 3, 2024
82c5f17
Use `parse_unchecked_source`
MichaReiser Jun 4, 2024
8f75726
Fix merge conflicts
MichaReiser Jun 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 0 additions & 24 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion crates/red_knot/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ tracing-subscriber = { workspace = true }
tracing-tree = { workspace = true }

[dev-dependencies]
textwrap = { version = "0.16.1" }
tempfile = { workspace = true }

[lints]
Expand Down
13 changes: 7 additions & 6 deletions crates/red_knot/src/lint.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ use std::time::Duration;

use ruff_python_ast::visitor::Visitor;
use ruff_python_ast::{ModModule, StringLiteral};
use ruff_python_parser::Parsed;

use crate::cache::KeyValueCache;
use crate::db::{LintDb, LintJar, QueryResult};
use crate::files::FileId;
use crate::module::ModuleName;
use crate::parse::{parse, Parsed};
use crate::parse::parse;
use crate::source::{source_text, Source};
use crate::symbols::{
resolve_global_symbol, symbol_table, Definition, GlobalSymbolId, SymbolId, SymbolTable,
Expand Down Expand Up @@ -40,7 +41,7 @@ pub(crate) fn lint_syntax(db: &dyn LintDb, file_id: FileId) -> QueryResult<Diagn
let parsed = parse(db.upcast(), *file_id)?;

if parsed.errors().is_empty() {
let ast = parsed.ast();
let ast = parsed.syntax();

let mut visitor = SyntaxLintVisitor {
diagnostics,
Expand Down Expand Up @@ -86,7 +87,7 @@ pub(crate) fn lint_semantic(db: &dyn LintDb, file_id: FileId) -> QueryResult<Dia
let context = SemanticLintContext {
file_id: *file_id,
source,
parsed,
parsed: &parsed,
symbols,
db,
diagnostics: RefCell::new(Vec::new()),
Expand Down Expand Up @@ -194,7 +195,7 @@ fn lint_bad_overrides(context: &SemanticLintContext) -> QueryResult<()> {
pub struct SemanticLintContext<'a> {
file_id: FileId,
source: Source,
parsed: Parsed,
parsed: &'a Parsed<ModModule>,
symbols: Arc<SymbolTable>,
db: &'a dyn LintDb,
diagnostics: RefCell<Vec<String>>,
Expand All @@ -209,8 +210,8 @@ impl<'a> SemanticLintContext<'a> {
self.file_id
}

pub fn ast(&self) -> &ModModule {
self.parsed.ast()
pub fn ast(&self) -> &'a ModModule {
self.parsed.syntax()
}

pub fn symbols(&self) -> &SymbolTable {
Expand Down
72 changes: 9 additions & 63 deletions crates/red_knot/src/parse.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,87 +1,33 @@
use std::ops::{Deref, DerefMut};
use std::sync::Arc;

use ruff_python_ast as ast;
use ruff_python_parser::{Mode, ParseError};
use ruff_text_size::{Ranged, TextRange};
use ruff_python_ast::ModModule;
use ruff_python_parser::Parsed;

use crate::cache::KeyValueCache;
use crate::db::{QueryResult, SourceDb};
use crate::files::FileId;
use crate::source::source_text;

#[derive(Debug, Clone, PartialEq)]
pub struct Parsed {
inner: Arc<ParsedInner>,
}

#[derive(Debug, PartialEq)]
struct ParsedInner {
ast: ast::ModModule,
errors: Vec<ParseError>,
}

impl Parsed {
fn new(ast: ast::ModModule, errors: Vec<ParseError>) -> Self {
Self {
inner: Arc::new(ParsedInner { ast, errors }),
}
}

pub(crate) fn from_text(text: &str) -> Self {
let result = ruff_python_parser::parse(text, Mode::Module);

let (module, errors) = match result {
Ok(parsed) => match parsed.into_syntax() {
ast::Mod::Module(module) => (module, vec![]),
ast::Mod::Expression(expression) => (
ast::ModModule {
range: expression.range(),
body: vec![ast::Stmt::Expr(ast::StmtExpr {
range: expression.range(),
value: expression.body,
})],
},
vec![],
),
},
Err(errors) => (
ast::ModModule {
range: TextRange::default(),
body: Vec::new(),
},
vec![errors],
),
};

Parsed::new(module, errors)
}

pub fn ast(&self) -> &ast::ModModule {
&self.inner.ast
}

pub fn errors(&self) -> &[ParseError] {
&self.inner.errors
}
}

#[tracing::instrument(level = "debug", skip(db))]
pub(crate) fn parse(db: &dyn SourceDb, file_id: FileId) -> QueryResult<Parsed> {
pub(crate) fn parse(db: &dyn SourceDb, file_id: FileId) -> QueryResult<Arc<Parsed<ModModule>>> {
let jar = db.jar()?;

jar.parsed.get(&file_id, |file_id| {
let source = source_text(db, *file_id)?;

Ok(Parsed::from_text(source.text()))
Ok(Arc::new(ruff_python_parser::parse_unchecked_source(
source.text(),
source.kind().into(),
)))
})
}

#[derive(Debug, Default)]
pub struct ParsedStorage(KeyValueCache<FileId, Parsed>);
pub struct ParsedStorage(KeyValueCache<FileId, Arc<Parsed<ModModule>>>);

impl Deref for ParsedStorage {
type Target = KeyValueCache<FileId, Parsed>;
type Target = KeyValueCache<FileId, Arc<Parsed<ModModule>>>;

fn deref(&self) -> &Self::Target {
&self.0
Expand Down
10 changes: 10 additions & 0 deletions crates/red_knot/src/source.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,16 @@ pub enum SourceKind {
IpyNotebook(Arc<Notebook>),
}

impl<'a> From<&'a SourceKind> for PySourceType {
fn from(value: &'a SourceKind) -> Self {
match value {
SourceKind::Python(_) => PySourceType::Python,
SourceKind::Stub(_) => PySourceType::Stub,
SourceKind::IpyNotebook(_) => PySourceType::Ipynb,
}
}
}

#[derive(Debug, Clone, PartialEq)]
pub struct Source {
kind: SourceKind,
Expand Down
43 changes: 23 additions & 20 deletions crates/red_knot/src/symbols.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ pub fn symbol_table(db: &dyn SemanticDb, file_id: FileId) -> QueryResult<Arc<Sym

jar.symbol_tables.get(&file_id, |_| {
let parsed = parse(db.upcast(), file_id)?;
Ok(Arc::from(SymbolTable::from_ast(parsed.ast())))
Ok(Arc::from(SymbolTable::from_ast(parsed.syntax())))
})
}

Expand Down Expand Up @@ -903,13 +903,16 @@ mod tests {
use crate::symbols::{ScopeKind, SymbolFlags, SymbolTable};

mod from_ast {
use crate::parse::Parsed;
use crate::symbols::{Definition, ScopeKind, SymbolId, SymbolIterator, SymbolTable};
use ruff_python_ast as ast;
use textwrap::dedent;
use ruff_python_ast::ModModule;
use ruff_python_parser::{Mode, Parsed};

use crate::symbols::{Definition, ScopeKind, SymbolId, SymbolIterator, SymbolTable};

fn parse(code: &str) -> Parsed {
Parsed::from_text(&dedent(code))
fn parse(code: &str) -> Parsed<ModModule> {
ruff_python_parser::parse_unchecked(code, Mode::Module)
.try_into_module()
.unwrap()
}

fn names<I>(it: SymbolIterator<I>) -> Vec<&str>
Expand All @@ -924,14 +927,14 @@ mod tests {
#[test]
fn empty() {
let parsed = parse("");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()).len(), 0);
}

#[test]
fn simple() {
let parsed = parse("x");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["x"]);
assert_eq!(
table
Expand All @@ -944,15 +947,15 @@ mod tests {
#[test]
fn annotation_only() {
let parsed = parse("x: int");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["int", "x"]);
// TODO record definition
}

#[test]
fn import() {
let parsed = parse("import foo");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["foo"]);
assert_eq!(
table
Expand All @@ -965,21 +968,21 @@ mod tests {
#[test]
fn import_sub() {
let parsed = parse("import foo.bar");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["foo"]);
}

#[test]
fn import_as() {
let parsed = parse("import foo.bar as baz");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["baz"]);
}

#[test]
fn import_from() {
let parsed = parse("from bar import foo");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["foo"]);
assert_eq!(
table
Expand All @@ -999,7 +1002,7 @@ mod tests {
#[test]
fn assign() {
let parsed = parse("x = foo");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["foo", "x"]);
assert_eq!(
table
Expand All @@ -1025,7 +1028,7 @@ mod tests {
y = 2
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["C", "y"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 1);
Expand All @@ -1050,7 +1053,7 @@ mod tests {
y = 2
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["func", "y"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 1);
Expand All @@ -1076,7 +1079,7 @@ mod tests {
y = 2
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["func"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 2);
Expand Down Expand Up @@ -1104,7 +1107,7 @@ mod tests {
x = 1
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["func"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 1);
Expand All @@ -1130,7 +1133,7 @@ mod tests {
x = 1
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["C"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 1);
Expand All @@ -1157,7 +1160,7 @@ mod tests {
#[test]
fn reachability_trivial() {
let parsed = parse("x = 1; x");
let ast = parsed.ast();
let ast = parsed.syntax();
let table = SymbolTable::from_ast(ast);
let x_sym = table
.root_symbol_id_by_name("x")
Expand Down
Loading
Loading