Use cursor based lexer

astral-sh · Jul 24, 2023 · a46b9c0 · a46b9c0
1 parent 8a7dcb7
commit a46b9c0
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 23 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -52,11 +52,17 @@ wsl = { version = "0.1.0" }
 # v1.0.1
 libcst = { git = "https://github.com/Instagram/LibCST.git", rev = "3cacca1a1029f05707e50703b49fe3dd860aa839", default-features = false }
 
-ruff_text_size = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "4d03b9b5b212fc869e4cfda151414438186a7779" }
-rustpython-ast = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "4d03b9b5b212fc869e4cfda151414438186a7779" , default-features = false, features = ["num-bigint"]}
-rustpython-format = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "4d03b9b5b212fc869e4cfda151414438186a7779", default-features = false, features = ["num-bigint"] }
-rustpython-literal = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "4d03b9b5b212fc869e4cfda151414438186a7779", default-features = false }
-rustpython-parser = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "4d03b9b5b212fc869e4cfda151414438186a7779" , default-features = false, features = ["full-lexer", "num-bigint"] }
+ruff_text_size = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "421639c0eabb3ff0358a8382cf098efe9f86ecda" }
+rustpython-ast = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "421639c0eabb3ff0358a8382cf098efe9f86ecda" }
+rustpython-format = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "421639c0eabb3ff0358a8382cf098efe9f86ecda" }
+rustpython-literal = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "421639c0eabb3ff0358a8382cf098efe9f86ecda" }
+rustpython-parser = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "421639c0eabb3ff0358a8382cf098efe9f86ecda" }
+
+#ruff_text_size = { path = "../RustPython-parser/ruff_text_size" }
+#rustpython-ast = { path = "../RustPython-parser/ast" , default-features = false}
+#rustpython-format = { path = "../RustPython-parser/format", default-features = false }
+#rustpython-literal = { path = "../RustPython-parser/literal", default-features = false }
+#rustpython-parser = { path = "../RustPython-parser/parser", default-features = false}
 
 [profile.release]
 lto = "fat"

diff --git a/crates/ruff/resources/test/fixtures/jupyter/not_json.ipynb b/crates/ruff/resources/test/fixtures/jupyter/not_json.ipynb
@@ -1 +1 @@
-broken "§=($/=(")
+broken "§=($/=()
diff --git a/crates/ruff/src/jupyter/notebook.rs b/crates/ruff/src/jupyter/notebook.rs
@@ -12,6 +12,8 @@ use serde_json::error::Category;
 use ruff_diagnostics::Diagnostic;
 use ruff_python_trivia::{NewlineWithTrailingNewline, UniversalNewlineIterator};
 use ruff_text_size::{TextRange, TextSize};
+use rustpython_parser::lexer::lex;
+use rustpython_parser::Mode;
 
 use crate::autofix::source_map::{SourceMap, SourceMarker};
 use crate::jupyter::index::JupyterIndex;
@@ -158,10 +160,7 @@ impl Notebook {
                                 )
                             })?;
                             // Check if tokenizing was successful and the file is non-empty
-                            if (ruff_rustpython::tokenize(&contents))
-                                .last()
-                                .map_or(true, Result::is_err)
-                            {
+                            if lex(&contents, Mode::Module).any(|result| result.is_err()) {
                                 Diagnostic::new(
                                     SyntaxError {
                                         message: format!(

diff --git a/crates/ruff_python_ast/Cargo.toml b/crates/ruff_python_ast/Cargo.toml
@@ -20,7 +20,6 @@ anyhow = { workspace = true }
 bitflags = { workspace = true }
 is-macro = { workspace = true }
 itertools = { workspace = true }
-log = { workspace = true }
 memchr = { workspace = true }
 num-bigint = { workspace = true }
 num-traits = { workspace = true }

diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml
@@ -24,7 +24,7 @@ ruff_python_ast = { path = "../crates/ruff_python_ast" }
 ruff_python_formatter = { path = "../crates/ruff_python_formatter" }
 similar = { version = "2.2.1" }
 
-rustpython-parser = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "4d03b9b5b212fc869e4cfda151414438186a7779" , default-features = false, features = ["full-lexer", "num-bigint"] }
+rustpython-parser = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "421639c0eabb3ff0358a8382cf098efe9f86ecda" }
 
 # Prevent this from interfering with workspaces
 [workspace]