feat(ast)! add source_text to Program (#6444)

crates/oxc/README.md

@@ -99,7 +99,7 @@ if panicked {
 let SemanticBuilderReturn {
     semantic,
     errors: semantic_errors,
-} = SemanticBuilder::new(source_text)
+} = SemanticBuilder::new()
     .with_check_syntax_error(true) // Enable extra syntax error checking
     .with_trivias(trivias)         // Pass comments for JSDoc parsing
     .with_build_jsdoc(true)        // Enable JSDoc parsing

crates/oxc/src/compiler.rs

@@ -139,19 +139,12 @@ pub trait CompilerInterface {
 
         /* Isolated Declarations */
         if let Some(options) = self.isolated_declaration_options() {
-            self.isolated_declaration(
-                options,
-                &allocator,
-                &program,
-                source_text,
-                source_path,
-                &trivias,
-            );
+            self.isolated_declaration(options, &allocator, &program, source_path, &trivias);
         }
 
         /* Semantic */
 
-        let mut semantic_return = self.semantic(&program, source_text, source_path);
+        let mut semantic_return = self.semantic(&program, source_path);
         if !semantic_return.errors.is_empty() {
             self.handle_errors(semantic_return.errors);
             return;
@@ -170,7 +163,6 @@ pub trait CompilerInterface {
                 &allocator,
                 &mut program,
                 source_path,
-                source_text,
                 &trivias,
                 symbols,
                 scopes,
@@ -218,7 +210,7 @@ pub trait CompilerInterface {
         /* Codegen */
 
         if let Some(options) = self.codegen_options() {
-            let ret = self.codegen(&program, source_text, source_path, &trivias, mangler, options);
+            let ret = self.codegen(&program, source_path, &trivias, mangler, options);
             self.after_codegen(ret);
         }
     }
@@ -232,13 +224,8 @@ pub trait CompilerInterface {
         Parser::new(allocator, source_text, source_type).with_options(self.parse_options()).parse()
     }
 
-    fn semantic<'a>(
-        &self,
-        program: &Program<'a>,
-        source_text: &'a str,
-        source_path: &Path,
-    ) -> SemanticBuilderReturn<'a> {
-        let mut builder = SemanticBuilder::new(source_text);
+    fn semantic<'a>(&self, program: &Program<'a>, source_path: &Path) -> SemanticBuilderReturn<'a> {
+        let mut builder = SemanticBuilder::new();
 
         if self.transform_options().is_some() {
             // Estimate transformer will triple scopes, symbols, references
@@ -257,16 +244,14 @@ pub trait CompilerInterface {
         options: IsolatedDeclarationsOptions,
         allocator: &'a Allocator,
         program: &Program<'a>,
-        source_text: &'a str,
         source_path: &Path,
         trivias: &Trivias,
     ) {
-        let ret =
-            IsolatedDeclarations::new(allocator, source_text, trivias, options).build(program);
+        let ret = IsolatedDeclarations::new(allocator, program.source_text, trivias, options)
+            .build(program);
         self.handle_errors(ret.errors);
         let ret = self.codegen(
             &ret.program,
-            source_text,
             source_path,
             trivias,
             None,
@@ -282,12 +267,11 @@ pub trait CompilerInterface {
         allocator: &'a Allocator,
         program: &mut Program<'a>,
         source_path: &Path,
-        source_text: &'a str,
         trivias: &Trivias,
         symbols: SymbolTable,
         scopes: ScopeTree,
     ) -> TransformerReturn {
-        Transformer::new(allocator, source_path, source_text, trivias.clone(), options)
+        Transformer::new(allocator, source_path, trivias.clone(), options)
             .build_with_symbols_and_scopes(symbols, scopes, program)
     }
 
@@ -304,10 +288,9 @@ pub trait CompilerInterface {
         Mangler::new().with_options(options).build(program)
     }
 
-    fn codegen<'a>(
+    fn codegen(
         &self,
-        program: &Program<'a>,
-        source_text: &'a str,
+        program: &Program<'_>,
         source_path: &Path,
         trivias: &Trivias,
         mangler: Option<Mangler>,
@@ -317,10 +300,10 @@ pub trait CompilerInterface {
         let mut codegen = CodeGenerator::new()
             .with_options(options)
             .with_mangler(mangler)
-            .enable_comment(source_text, trivias.clone(), comment_options);
+            .enable_comment(program.source_text, trivias.clone(), comment_options);
         if self.enable_sourcemap() {
-            codegen =
-                codegen.enable_source_map(source_path.to_string_lossy().as_ref(), source_text);
+            codegen = codegen
+                .enable_source_map(source_path.to_string_lossy().as_ref(), program.source_text);
         }
         codegen.build(program)
     }

crates/oxc_ast/src/ast/js.rs

@@ -39,6 +39,8 @@ pub struct Program<'a> {
     #[serde(flatten)]
     pub span: Span,
     pub source_type: SourceType,
+    #[serde(skip)]
+    pub source_text: &'a str,
     pub hashbang: Option<Hashbang<'a>>,
     pub directives: Vec<'a, Directive<'a>>,
     pub body: Vec<'a, Statement<'a>>,

crates/oxc_ast/src/ast_impl/js.rs

@@ -1,7 +1,7 @@
 use std::{borrow::Cow, cell::Cell, fmt};
 
 use oxc_allocator::{Box, FromIn, Vec};
-use oxc_span::{Atom, GetSpan, SourceType, Span};
+use oxc_span::{Atom, GetSpan, Span};
 use oxc_syntax::{
     operator::UnaryOperator,
     reference::ReferenceId,
@@ -28,18 +28,6 @@ export interface FormalParameterRest extends Span {
 }
 "#;
 
-impl<'a> Program<'a> {
-    pub fn new(
-        span: Span,
-        source_type: SourceType,
-        directives: Vec<'a, Directive<'a>>,
-        hashbang: Option<Hashbang<'a>>,
-        body: Vec<'a, Statement<'a>>,
-    ) -> Self {
-        Self { span, source_type, directives, hashbang, body, scope_id: Cell::default() }
-    }
-}
-
 impl<'a> Program<'a> {
     pub fn is_empty(&self) -> bool {
         self.body.is_empty() && self.directives.is_empty()

crates/oxc_ast/src/generated/assert_layouts.rs

@@ -55,14 +55,15 @@ const _: () = {
     assert!(offset_of!(StringLiteral, span) == 0usize);
     assert!(offset_of!(StringLiteral, value) == 8usize);
 
-    assert!(size_of::<Program>() == 112usize);
+    assert!(size_of::<Program>() == 128usize);
     assert!(align_of::<Program>() == 8usize);
     assert!(offset_of!(Program, span) == 0usize);
     assert!(offset_of!(Program, source_type) == 8usize);
-    assert!(offset_of!(Program, hashbang) == 16usize);
-    assert!(offset_of!(Program, directives) == 40usize);
-    assert!(offset_of!(Program, body) == 72usize);
-    assert!(offset_of!(Program, scope_id) == 104usize);
+    assert!(offset_of!(Program, source_text) == 16usize);
+    assert!(offset_of!(Program, hashbang) == 32usize);
+    assert!(offset_of!(Program, directives) == 56usize);
+    assert!(offset_of!(Program, body) == 88usize);
+    assert!(offset_of!(Program, scope_id) == 120usize);
 
     assert!(size_of::<Expression>() == 16usize);
     assert!(align_of::<Expression>() == 8usize);
@@ -1597,14 +1598,15 @@ const _: () = {
     assert!(offset_of!(StringLiteral, span) == 0usize);
     assert!(offset_of!(StringLiteral, value) == 8usize);
 
-    assert!(size_of::<Program>() == 64usize);
+    assert!(size_of::<Program>() == 72usize);
     assert!(align_of::<Program>() == 4usize);
     assert!(offset_of!(Program, span) == 0usize);
     assert!(offset_of!(Program, source_type) == 8usize);
-    assert!(offset_of!(Program, hashbang) == 12usize);
-    assert!(offset_of!(Program, directives) == 28usize);
-    assert!(offset_of!(Program, body) == 44usize);
-    assert!(offset_of!(Program, scope_id) == 60usize);
+    assert!(offset_of!(Program, source_text) == 12usize);
+    assert!(offset_of!(Program, hashbang) == 20usize);
+    assert!(offset_of!(Program, directives) == 36usize);
+    assert!(offset_of!(Program, body) == 52usize);
+    assert!(offset_of!(Program, scope_id) == 68usize);
 
     assert!(size_of::<Expression>() == 8usize);
     assert!(align_of::<Expression>() == 4usize);

crates/oxc_ast/src/generated/ast_builder.rs

@@ -221,19 +221,32 @@ impl<'a> AstBuilder<'a> {
     /// ## Parameters
     /// - span: The [`Span`] covering this node
     /// - source_type
+    /// - source_text
     /// - hashbang
     /// - directives
     /// - body
     #[inline]
-    pub fn program(
+    pub fn program<S>(
         self,
         span: Span,
         source_type: SourceType,
+        source_text: S,
         hashbang: Option<Hashbang<'a>>,
         directives: Vec<'a, Directive<'a>>,
         body: Vec<'a, Statement<'a>>,
-    ) -> Program<'a> {
-        Program { span, source_type, hashbang, directives, body, scope_id: Default::default() }
+    ) -> Program<'a>
+    where
+        S: IntoIn<'a, &'a str>,
+    {
+        Program {
+            span,
+            source_type,
+            source_text: source_text.into_in(self.allocator),
+            hashbang,
+            directives,
+            body,
+            scope_id: Default::default(),
+        }
     }
 
     /// Builds a [`Program`] and stores it in the memory arena.
@@ -243,19 +256,27 @@ impl<'a> AstBuilder<'a> {
     /// ## Parameters
     /// - span: The [`Span`] covering this node
     /// - source_type
+    /// - source_text
     /// - hashbang
     /// - directives
     /// - body
     #[inline]
-    pub fn alloc_program(
+    pub fn alloc_program<S>(
         self,
         span: Span,
         source_type: SourceType,
+        source_text: S,
         hashbang: Option<Hashbang<'a>>,
         directives: Vec<'a, Directive<'a>>,
         body: Vec<'a, Statement<'a>>,
-    ) -> Box<'a, Program<'a>> {
-        Box::new_in(self.program(span, source_type, hashbang, directives, body), self.allocator)
+    ) -> Box<'a, Program<'a>>
+    where
+        S: IntoIn<'a, &'a str>,
+    {
+        Box::new_in(
+            self.program(span, source_type, source_text, hashbang, directives, body),
+            self.allocator,
+        )
     }
 
     /// Build a [`Expression::BooleanLiteral`]

crates/oxc_ast/src/generated/derive_clone_in.rs

@@ -112,6 +112,7 @@ impl<'old_alloc, 'new_alloc> CloneIn<'new_alloc> for Program<'old_alloc> {
         Program {
             span: CloneIn::clone_in(&self.span, allocator),
             source_type: CloneIn::clone_in(&self.source_type, allocator),
+            source_text: CloneIn::clone_in(&self.source_text, allocator),
             hashbang: CloneIn::clone_in(&self.hashbang, allocator),
             directives: CloneIn::clone_in(&self.directives, allocator),
             body: CloneIn::clone_in(&self.body, allocator),

crates/oxc_ast/src/generated/derive_content_eq.rs

@@ -92,6 +92,7 @@ impl<'a> ContentEq for StringLiteral<'a> {
 impl<'a> ContentEq for Program<'a> {
     fn content_eq(&self, other: &Self) -> bool {
         ContentEq::content_eq(&self.source_type, &other.source_type)
+            && ContentEq::content_eq(&self.source_text, &other.source_text)
             && ContentEq::content_eq(&self.hashbang, &other.hashbang)
             && ContentEq::content_eq(&self.directives, &other.directives)
             && ContentEq::content_eq(&self.body, &other.body)

crates/oxc_ast/src/generated/derive_content_hash.rs

@@ -70,6 +70,7 @@ impl<'a> ContentHash for StringLiteral<'a> {
 impl<'a> ContentHash for Program<'a> {
     fn content_hash<H: Hasher>(&self, state: &mut H) {
         ContentHash::content_hash(&self.source_type, state);
+        ContentHash::content_hash(&self.source_text, state);
         ContentHash::content_hash(&self.hashbang, state);
         ContentHash::content_hash(&self.directives, state);
         ContentHash::content_hash(&self.body, state);

crates/oxc_isolated_declarations/src/lib.rs

@@ -92,7 +92,8 @@ impl<'a> IsolatedDeclarations<'a> {
         let source_type = SourceType::d_ts();
         let directives = self.ast.vec();
         let stmts = self.transform_program(program);
-        let program = self.ast.program(SPAN, source_type, None, directives, stmts);
+        let program =
+            self.ast.program(SPAN, source_type, program.source_text, None, directives, stmts);
         IsolatedDeclarationsReturn { program, errors: self.take_errors() }
     }
 

crates/oxc_language_server/src/linter.rs

@@ -260,7 +260,7 @@ impl IsolatedLintHandler {
             };
 
             let program = allocator.alloc(ret.program);
-            let semantic_ret = SemanticBuilder::new(javascript_source_text)
+            let semantic_ret = SemanticBuilder::new()
                 .with_cfg(true)
                 .with_trivias(ret.trivias)
                 .with_check_syntax_error(true)

crates/oxc_linter/examples/linter.rs

@@ -30,7 +30,7 @@ fn main() -> std::io::Result<()> {
     }
 
     let program = allocator.alloc(ret.program);
-    let semantic_ret = SemanticBuilder::new(&source_text).with_trivias(ret.trivias).build(program);
+    let semantic_ret = SemanticBuilder::new().with_trivias(ret.trivias).build(program);
 
     let mut errors: Vec<OxcDiagnostic> = vec![];
 

crates/oxc_linter/src/service/runtime.rs

@@ -208,7 +208,7 @@ impl Runtime {
 
         // Build the module record to unblock other threads from waiting for too long.
         // The semantic model is not built at this stage.
-        let semantic_builder = SemanticBuilder::new(source_text)
+        let semantic_builder = SemanticBuilder::new()
             .with_cfg(true)
             .with_trivias(trivias)
             .with_build_jsdoc(true)

crates/oxc_linter/src/utils/jest.rs

@@ -318,7 +318,7 @@ mod test {
         let source_type = SourceType::default();
         let parser_ret = Parser::new(&allocator, "", source_type).parse();
         let program = allocator.alloc(parser_ret.program);
-        let semantic_ret = SemanticBuilder::new("").with_cfg(true).build(program).semantic;
+        let semantic_ret = SemanticBuilder::new().with_cfg(true).build(program).semantic;
         let semantic_ret = Rc::new(semantic_ret);
 
         let build_ctx = |path: &'static str| {

crates/oxc_mangler/src/lib.rs

@@ -82,7 +82,7 @@ impl Mangler {
 
     #[must_use]
     pub fn build<'a>(mut self, program: &'a Program<'a>) -> Mangler {
-        let semantic = SemanticBuilder::new("").build(program).semantic;
+        let semantic = SemanticBuilder::new().build(program).semantic;
 
         // Mangle the symbol table by computing slots from the scope tree.
         // A slot is the occurrence index of a binding identifier inside a scope.

crates/oxc_minifier/src/compressor.rs

@@ -24,7 +24,7 @@ impl<'a> Compressor<'a> {
 
     pub fn build(self, program: &mut Program<'a>) {
         let (symbols, scopes) =
-            SemanticBuilder::new("").build(program).semantic.into_symbol_table_and_scope_tree();
+            SemanticBuilder::new().build(program).semantic.into_symbol_table_and_scope_tree();
         self.build_with_symbols_and_scopes(symbols, scopes, program);
     }
 

crates/oxc_minifier/src/tester.rs

@@ -28,7 +28,7 @@ fn run<'a, P: CompressorPass<'a>>(
 
     if let Some(pass) = pass {
         let (symbols, scopes) =
-            SemanticBuilder::new("").build(&program).semantic.into_symbol_table_and_scope_tree();
+            SemanticBuilder::new().build(&program).semantic.into_symbol_table_and_scope_tree();
         let mut ctx = TraverseCtx::new(scopes, symbols, allocator);
         RemoveSyntax::new(CompressOptions::all_false()).build(&mut program, &mut ctx);
         pass.build(&mut program, &mut ctx);

crates/oxc_parser/src/lib.rs

@@ -404,6 +404,7 @@ impl<'a> ParserImpl<'a> {
                 let program = self.ast.program(
                     Span::default(),
                     self.source_type,
+                    self.source_text,
                     None,
                     self.ast.vec(),
                     self.ast.vec(),
@@ -450,7 +451,14 @@ impl<'a> ParserImpl<'a> {
         self.set_source_type_to_script_if_unambiguous();
 
         let span = Span::new(0, self.source_text.len() as u32);
-        Ok(self.ast.program(span, self.source_type, hashbang, directives, statements))
+        Ok(self.ast.program(
+            span,
+            self.source_type,
+            self.source_text,
+            hashbang,
+            directives,
+            statements,
+        ))
     }
 
     fn default_context(source_type: SourceType, options: ParseOptions) -> Context {

crates/oxc_semantic/examples/cfg.rs

@@ -55,7 +55,7 @@ fn main() -> std::io::Result<()> {
     std::fs::write(ast_file_path, format!("{:#?}", &program))?;
     println!("Wrote AST to: {}", &ast_file_name);
 
-    let semantic = SemanticBuilder::new(&source_text)
+    let semantic = SemanticBuilder::new()
         .with_check_syntax_error(true)
         .with_trivias(parser_ret.trivias)
         .with_cfg(true)