diff --git a/Cargo.lock b/Cargo.lock index f8c99ea4eb1ea..ff29fd4c24957 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5182,6 +5182,7 @@ dependencies = [ "num-bigint", "phf", "scoped-tls", + "serde", "string_enum", "swc_atoms", "swc_common", @@ -6361,6 +6362,9 @@ dependencies = [ "shared_child", "struct_iterable", "svix-ksuid", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", "sysinfo", "tabwriter", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 80484ff05f4b2..ad2590030e909 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,6 +146,10 @@ smallvec = { version = "1.13.1", features = [ "union", "const_new", ] } +swc_common = "0.37.5" +swc_ecma_ast = "0.118.2" +swc_ecma_parser = "0.149.1" +swc_ecma_visit = "0.104.8" syn = "1.0.107" tempfile = "3.3.0" test-case = "3.0.0" diff --git a/crates/turbo-trace/Cargo.toml b/crates/turbo-trace/Cargo.toml index 5eeb45bfc5c34..08e15986dca9a 100644 --- a/crates/turbo-trace/Cargo.toml +++ b/crates/turbo-trace/Cargo.toml @@ -9,10 +9,10 @@ camino.workspace = true clap = { version = "4.5.17", features = ["derive"] } miette = { workspace = true, features = ["fancy"] } oxc_resolver = "1.11.0" -swc_common = "0.37.5" -swc_ecma_ast = "0.118.2" -swc_ecma_parser = "0.149.1" -swc_ecma_visit = "0.104.8" +swc_common = { workspace = true } +swc_ecma_ast = { workspace = true } +swc_ecma_parser = { workspace = true } +swc_ecma_visit = { workspace = true } thiserror = { workspace = true } turbopath = { workspace = true } diff --git a/crates/turbo-trace/src/main.rs b/crates/turbo-trace/src/main.rs index 5c5f10473ede9..f9b13a344a5c8 100644 --- a/crates/turbo-trace/src/main.rs +++ b/crates/turbo-trace/src/main.rs @@ -13,6 +13,8 @@ struct Args { #[clap(long)] ts_config: Option, files: Vec, + #[clap(long)] + depth: Option, } fn main() -> Result<(), PathError> { @@ -32,7 +34,7 @@ fn main() -> Result<(), PathError> { let tracer = Tracer::new(abs_cwd, files, args.ts_config); - let result = tracer.trace(); + let result = tracer.trace(args.depth); if !result.errors.is_empty() { for error in &result.errors { @@ -40,7 +42,7 @@ fn main() -> Result<(), PathError> { } std::process::exit(1); } else { - for file in &result.files { + for file in result.files.keys() { println!("{}", file); } } diff --git a/crates/turbo-trace/src/tracer.rs b/crates/turbo-trace/src/tracer.rs index bab720d6261de..f94e085ebdb6d 100644 --- a/crates/turbo-trace/src/tracer.rs +++ b/crates/turbo-trace/src/tracer.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, fs, rc::Rc}; +use std::{collections::HashMap, fs, rc::Rc}; use camino::Utf8PathBuf; use miette::{Diagnostic, NamedSource, SourceSpan}; @@ -14,9 +14,13 @@ use turbopath::{AbsoluteSystemPathBuf, PathError}; use crate::import_finder::ImportFinder; +#[derive(Default)] +pub struct SeenFile { + pub ast: Option, +} + pub struct Tracer { - files: Vec, - seen: HashSet, + files: Vec<(AbsoluteSystemPathBuf, usize)>, ts_config: Option, source_map: Rc, } @@ -40,7 +44,7 @@ pub enum TraceError { pub struct TraceResult { pub errors: Vec, - pub files: HashSet, + pub files: HashMap, } impl Tracer { @@ -52,22 +56,22 @@ impl Tracer { let ts_config = ts_config.map(|ts_config| AbsoluteSystemPathBuf::from_unknown(&cwd, ts_config)); - let seen = HashSet::new(); + let files = files.into_iter().map(|file| (file, 0)).collect::>(); Self { files, - seen, ts_config, source_map: Rc::new(SourceMap::default()), } } - pub fn trace(mut self) -> TraceResult { + pub fn trace(mut self, max_depth: Option) -> TraceResult { let mut options = ResolveOptions::default() .with_builtin_modules(true) .with_force_extension(EnforceExtension::Disabled) .with_extension(".ts") .with_extension(".tsx"); + if let Some(ts_config) = self.ts_config.take() { options.tsconfig = Some(TsconfigOptions { config_file: ts_config.into(), @@ -77,17 +81,24 @@ impl Tracer { let resolver = Resolver::new(options); let mut errors = vec![]; + let mut seen: HashMap = HashMap::new(); - while let Some(file_path) = self.files.pop() { + while let Some((file_path, file_depth)) = self.files.pop() { if matches!(file_path.extension(), Some("json") | Some("css")) { continue; } - if self.seen.contains(&file_path) { + if seen.contains_key(&file_path) { continue; } - self.seen.insert(file_path.clone()); + if let Some(max_depth) = max_depth { + if file_depth > max_depth { + continue; + } + } + + let entry = seen.entry(file_path.clone()).or_default(); // Read the file content let Ok(file_content) = fs::read_to_string(&file_path) else { @@ -135,6 +146,8 @@ impl Tracer { let mut finder = ImportFinder::default(); module.visit_with(&mut finder); + entry.ast = Some(module); + // Convert found imports/requires to absolute paths and add them to files to // visit for (import, span) in finder.imports() { @@ -144,7 +157,7 @@ impl Tracer { }; match resolver.resolve(file_dir, import) { Ok(resolved) => match resolved.into_path_buf().try_into() { - Ok(path) => self.files.push(path), + Ok(path) => self.files.push((path, file_depth + 1)), Err(err) => { errors.push(TraceError::PathEncoding(err)); } @@ -163,7 +176,7 @@ impl Tracer { } TraceResult { - files: self.seen, + files: seen, errors, } } diff --git a/crates/turborepo-lib/Cargo.toml b/crates/turborepo-lib/Cargo.toml index a88c7cd83d6c3..56816b40a639c 100644 --- a/crates/turborepo-lib/Cargo.toml +++ b/crates/turborepo-lib/Cargo.toml @@ -99,6 +99,9 @@ sha2 = { workspace = true } shared_child = "1.0.0" struct_iterable = "0.1.1" svix-ksuid = { version = "0.7.0", features = ["serde"] } +swc_common = { workspace = true } +swc_ecma_ast = { workspace = true, features = ["serde-impl"] } +swc_ecma_parser = { workspace = true } sysinfo = "0.27.7" tabwriter = "1.3.0" thiserror = "1.0.38" diff --git a/crates/turborepo-lib/src/query/file.rs b/crates/turborepo-lib/src/query/file.rs index b24e28508a8a0..78a23965cc748 100644 --- a/crates/turborepo-lib/src/query/file.rs +++ b/crates/turborepo-lib/src/query/file.rs @@ -2,6 +2,8 @@ use std::sync::Arc; use async_graphql::{Object, SimpleObject}; use itertools::Itertools; +use swc_ecma_ast::EsVersion; +use swc_ecma_parser::{EsSyntax, Syntax, TsSyntax}; use turbo_trace::Tracer; use turbopath::AbsoluteSystemPathBuf; @@ -13,11 +15,56 @@ use crate::{ pub struct File { run: Arc, path: AbsoluteSystemPathBuf, + ast: Option, } impl File { pub fn new(run: Arc, path: AbsoluteSystemPathBuf) -> Self { - Self { run, path } + Self { + run, + path, + ast: None, + } + } + + pub fn with_ast(mut self, ast: Option) -> Self { + self.ast = ast; + + self + } + + fn parse_file(&self) -> Result { + let contents = self.path.read_to_string()?; + let source_map = swc_common::SourceMap::default(); + let file = source_map.new_source_file( + swc_common::FileName::Custom(self.path.to_string()).into(), + contents.clone(), + ); + let syntax = if self.path.extension() == Some("ts") || self.path.extension() == Some("tsx") + { + Syntax::Typescript(TsSyntax { + tsx: self.path.extension() == Some("tsx"), + decorators: true, + ..Default::default() + }) + } else { + Syntax::Es(EsSyntax { + jsx: self.path.ends_with(".jsx"), + ..Default::default() + }) + }; + let comments = swc_common::comments::SingleThreadedComments::default(); + let mut errors = Vec::new(); + let module = swc_ecma_parser::parse_file_as_module( + &file, + syntax, + EsVersion::EsNext, + Some(&comments), + &mut errors, + ) + .map_err(Error::Parse)?; + + Ok(module) } } @@ -73,8 +120,8 @@ impl TraceResult { files: result .files .into_iter() - .sorted() - .map(|path| File::new(run.clone(), path)) + .sorted_by(|a, b| a.0.cmp(&b.0)) + .map(|(path, file)| File::new(run.clone(), path).with_ast(file.ast)) .collect(), errors: result.errors.into_iter().map(|e| e.into()).collect(), } @@ -100,16 +147,24 @@ impl File { Ok(self.path.to_string()) } - async fn dependencies(&self) -> TraceResult { + async fn dependencies(&self, depth: Option) -> TraceResult { let tracer = Tracer::new( self.run.repo_root().to_owned(), vec![self.path.clone()], None, ); - let mut result = tracer.trace(); + let mut result = tracer.trace(depth); // Remove the file itself from the result result.files.remove(&self.path); TraceResult::new(result, self.run.clone()) } + + async fn ast(&self) -> Option { + if let Some(ast) = &self.ast { + serde_json::to_value(ast).ok() + } else { + serde_json::to_value(&self.parse_file().ok()?).ok() + } + } } diff --git a/crates/turborepo-lib/src/query/mod.rs b/crates/turborepo-lib/src/query/mod.rs index fa61ea618014e..f6add1f4dd7ef 100644 --- a/crates/turborepo-lib/src/query/mod.rs +++ b/crates/turborepo-lib/src/query/mod.rs @@ -49,6 +49,8 @@ pub enum Error { #[error(transparent)] #[diagnostic(transparent)] Resolution(#[from] crate::run::scope::filter::ResolutionError), + #[error("failed to parse file: {0:?}")] + Parse(swc_ecma_parser::error::Error), } pub struct RepositoryQuery { diff --git a/turborepo-tests/integration/fixtures/turbo_trace/bar.js b/turborepo-tests/integration/fixtures/turbo_trace/bar.js new file mode 100644 index 0000000000000..a85f7dedb21d8 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/bar.js @@ -0,0 +1,3 @@ +export default function bar() { + console.log("bar"); +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace/foo.js b/turborepo-tests/integration/fixtures/turbo_trace/foo.js index de9fc706c8115..6bfab2b18bb17 100644 --- a/turborepo-tests/integration/fixtures/turbo_trace/foo.js +++ b/turborepo-tests/integration/fixtures/turbo_trace/foo.js @@ -1,3 +1,5 @@ +import { bar } from "./bar"; + export default function foo() { if (!process.env.IS_CI) { return "bar"; diff --git a/turborepo-tests/integration/tests/turbo-trace.t b/turborepo-tests/integration/tests/turbo-trace.t index 24569c5c761c9..1cced6353923b 100644 --- a/turborepo-tests/integration/tests/turbo-trace.t +++ b/turborepo-tests/integration/tests/turbo-trace.t @@ -20,6 +20,9 @@ Setup "dependencies": { "files": { "items": [ + { + "path": "bar.js" + }, { "path": "button.tsx" }, @@ -97,3 +100,337 @@ Trace file with invalid import } } +Get AST from file + $ ${TURBO} query "query { file(path: \"main.ts\") { path ast } }" + WARNING query command is experimental and may change in the future + { + "data": { + "file": { + "path": "main.ts", + "ast": { + "type": "Module", + "span": { + "start": 1, + "end": 169 + }, + "body": [ + { + "type": "ImportDeclaration", + "span": { + "start": 1, + "end": 35 + }, + "specifiers": [ + { + "type": "ImportSpecifier", + "span": { + "start": 10, + "end": 16 + }, + "local": { + "type": "Identifier", + "span": { + "start": 10, + "end": 16 + }, + "ctxt": 0, + "value": "Button", + "optional": false + }, + "imported": null, + "isTypeOnly": false + } + ], + "source": { + "type": "StringLiteral", + "span": { + "start": 24, + "end": 34 + }, + "value": "./button", + "raw": "\"./button\"" + }, + "typeOnly": false, + "with": null, + "phase": "evaluation" + }, + { + "type": "ImportDeclaration", + "span": { + "start": 36, + "end": 60 + }, + "specifiers": [ + { + "type": "ImportDefaultSpecifier", + "span": { + "start": 43, + "end": 46 + }, + "local": { + "type": "Identifier", + "span": { + "start": 43, + "end": 46 + }, + "ctxt": 0, + "value": "foo", + "optional": false + } + } + ], + "source": { + "type": "StringLiteral", + "span": { + "start": 52, + "end": 59 + }, + "value": "./foo", + "raw": "\"./foo\"" + }, + "typeOnly": false, + "with": null, + "phase": "evaluation" + }, + { + "type": "ImportDeclaration", + "span": { + "start": 61, + "end": 96 + }, + "specifiers": [ + { + "type": "ImportDefaultSpecifier", + "span": { + "start": 68, + "end": 74 + }, + "local": { + "type": "Identifier", + "span": { + "start": 68, + "end": 74 + }, + "ctxt": 0, + "value": "repeat", + "optional": false + } + } + ], + "source": { + "type": "StringLiteral", + "span": { + "start": 80, + "end": 95 + }, + "value": "repeat-string", + "raw": "\"repeat-string\"" + }, + "typeOnly": false, + "with": null, + "phase": "evaluation" + }, + { + "type": "VariableDeclaration", + "span": { + "start": 98, + "end": 126 + }, + "ctxt": 0, + "kind": "const", + "declare": false, + "declarations": [ + { + "type": "VariableDeclarator", + "span": { + "start": 104, + "end": 125 + }, + "id": { + "type": "Identifier", + "span": { + "start": 104, + "end": 110 + }, + "ctxt": 0, + "value": "button", + "optional": false, + "typeAnnotation": null + }, + "init": { + "type": "NewExpression", + "span": { + "start": 113, + "end": 125 + }, + "ctxt": 0, + "callee": { + "type": "Identifier", + "span": { + "start": 117, + "end": 123 + }, + "ctxt": 0, + "value": "Button", + "optional": false + }, + "arguments": [], + "typeArguments": null + }, + "definite": false + } + ] + }, + { + "type": "ExpressionStatement", + "span": { + "start": 128, + "end": 144 + }, + "expression": { + "type": "CallExpression", + "span": { + "start": 128, + "end": 143 + }, + "ctxt": 0, + "callee": { + "type": "MemberExpression", + "span": { + "start": 128, + "end": 141 + }, + "object": { + "type": "Identifier", + "span": { + "start": 128, + "end": 134 + }, + "ctxt": 0, + "value": "button", + "optional": false + }, + "property": { + "type": "Identifier", + "span": { + "start": 135, + "end": 141 + }, + "value": "render" + } + }, + "arguments": [], + "typeArguments": null + } + }, + { + "type": "ExpressionStatement", + "span": { + "start": 145, + "end": 162 + }, + "expression": { + "type": "CallExpression", + "span": { + "start": 145, + "end": 161 + }, + "ctxt": 0, + "callee": { + "type": "Identifier", + "span": { + "start": 145, + "end": 151 + }, + "ctxt": 0, + "value": "repeat", + "optional": false + }, + "arguments": [ + { + "spread": null, + "expression": { + "type": "StringLiteral", + "span": { + "start": 152, + "end": 157 + }, + "value": "foo", + "raw": "\"foo\"" + } + }, + { + "spread": null, + "expression": { + "type": "NumericLiteral", + "span": { + "start": 159, + "end": 160 + }, + "value": 5.0, + "raw": "5" + } + } + ], + "typeArguments": null + } + }, + { + "type": "ExpressionStatement", + "span": { + "start": 163, + "end": 169 + }, + "expression": { + "type": "CallExpression", + "span": { + "start": 163, + "end": 168 + }, + "ctxt": 0, + "callee": { + "type": "Identifier", + "span": { + "start": 163, + "end": 166 + }, + "ctxt": 0, + "value": "foo", + "optional": false + }, + "arguments": [], + "typeArguments": null + } + } + ], + "interpreter": null + } + } + } + } + +Set depth for dependencies + $ ${TURBO} query "query { file(path: \"main.ts\") { path dependencies(depth: 1) { files { items { path } } } } }" + WARNING query command is experimental and may change in the future + { + "data": { + "file": { + "path": "main.ts", + "dependencies": { + "files": { + "items": [ + { + "path": "button.tsx" + }, + { + "path": "foo.js" + }, + { + "path": "node_modules(\/|\\\\)repeat-string(\/|\\\\)index.js" (re) + } + ] + } + } + } + } + } \ No newline at end of file